{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TODAY'S DATE:\n",
      "Tue Aug  6 07:38:18 PDT 2019\n",
      "------------\n",
      "\n",
      "Distributor ID:\tUbuntu\n",
      "Description:\tUbuntu 16.04.6 LTS\n",
      "Release:\t16.04\n",
      "Codename:\txenial\n",
      "\n",
      "------------\n",
      "HOSTNAME: \n",
      "roadrunner\n",
      "\n",
      "------------\n",
      "Computer Specs:\n",
      "\n",
      "Architecture:          x86_64\n",
      "CPU op-mode(s):        32-bit, 64-bit\n",
      "Byte Order:            Little Endian\n",
      "CPU(s):                16\n",
      "On-line CPU(s) list:   0-15\n",
      "Thread(s) per core:    2\n",
      "Core(s) per socket:    4\n",
      "Socket(s):             2\n",
      "NUMA node(s):          1\n",
      "Vendor ID:             GenuineIntel\n",
      "CPU family:            6\n",
      "Model:                 26\n",
      "Model name:            Intel(R) Xeon(R) CPU           E5520  @ 2.27GHz\n",
      "Stepping:              5\n",
      "CPU MHz:               2394.000\n",
      "CPU max MHz:           2394.0000\n",
      "CPU min MHz:           1596.0000\n",
      "BogoMIPS:              4521.81\n",
      "Virtualization:        VT-x\n",
      "L1d cache:             32K\n",
      "L1i cache:             32K\n",
      "L2 cache:              256K\n",
      "L3 cache:              8192K\n",
      "NUMA node0 CPU(s):     0-15\n",
      "Flags:                 fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm dca sse4_1 sse4_2 popcnt lahf_lm ssbd ibrs ibpb stibp pti tpr_shadow vnmi flexpriority ept vpid dtherm ida flush_l1d\n",
      "\n",
      "------------\n",
      "\n",
      "Memory Specs\n",
      "\n",
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:            47G        1.3G         43G        443M        2.8G         44G\n",
      "Swap:           47G          0B         47G\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No LSB modules are available.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "echo \"TODAY'S DATE:\"\n",
    "date\n",
    "echo \"------------\"\n",
    "echo \"\"\n",
    "#Display operating system info\n",
    "lsb_release -a\n",
    "echo \"\"\n",
    "echo \"------------\"\n",
    "echo \"HOSTNAME: \"; hostname \n",
    "echo \"\"\n",
    "echo \"------------\"\n",
    "echo \"Computer Specs:\"\n",
    "echo \"\"\n",
    "lscpu\n",
    "echo \"\"\n",
    "echo \"------------\"\n",
    "echo \"\"\n",
    "echo \"Memory Specs\"\n",
    "echo \"\"\n",
    "free -mh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: wd=/home/sam/analyses/20190805_Pgenerosa_v070_repeatmasker_all\n",
      "env: fasta_url=https://owl.fish.washington.edu/halfshell/genomic-databank/Pgenerosa_v070.fa\n",
      "env: fasta=Pgenerosa_v070.fa\n",
      "env: repeat_masker=/home/shared/RepeatMasker-4.0.7/RepeatMasker\n",
      "env: cpus=16\n",
      "env: checksum=de0958fec4b9f8845babf3717ce7168c\n"
     ]
    }
   ],
   "source": [
    "%env wd=/home/sam/analyses/20190805_Pgenerosa_v070_repeatmasker_all\n",
    "%env fasta_url=https://owl.fish.washington.edu/halfshell/genomic-databank/Pgenerosa_v070.fa\n",
    "%env fasta=Pgenerosa_v070.fa\n",
    "%env repeat_masker=/home/shared/RepeatMasker-4.0.7/RepeatMasker\n",
    "%env cpus=16\n",
    "# Checksum taken from https://github.com/RobertsLab/resources/wiki/Genomic-Resources\n",
    "%env checksum=de0958fec4b9f8845babf3717ce7168c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create necessary directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "mkdir --parents ${wd}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Download _Crassostrea gigas genome FastA file\n",
    "\n",
    "Info on FastA file is here: [https://github.com/RobertsLab/resources/wiki/Genomic-Resources#genome](https://github.com/RobertsLab/resources/wiki/Genomic-Resources#genome)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "receiving incremental file list\n",
      "Pgenerosa_v070.fa\n",
      "\r",
      "              0   0%    0.00kB/s    0:00:00  \r",
      "     22,708,224   1%   21.66MB/s    0:01:40  \r",
      "     53,870,592   2%   25.70MB/s    0:01:23  \r",
      "     84,017,152   3%   26.73MB/s    0:01:19  \r",
      "    115,179,520   5%   27.47MB/s    0:01:15  \r",
      "    146,243,584   6%   29.29MB/s    0:01:10  \r",
      "    159,088,640   7%   24.75MB/s    0:01:22  \r",
      "    177,209,344   7%   21.92MB/s    0:01:32  \r",
      "    202,866,688   9%   20.48MB/s    0:01:37  \r",
      "    218,726,400   9%   17.03MB/s    0:01:56  \r",
      "    232,488,960  10%   17.29MB/s    0:01:53  \r",
      "    257,196,032  11%   18.84MB/s    0:01:43  \r",
      "    271,056,896  12%   13.06MB/s    0:02:27  \r",
      "    299,696,128  13%   15.51MB/s    0:02:02  \r",
      "    323,289,088  14%   17.47MB/s    0:01:47  \r",
      "    343,638,016  15%   16.40MB/s    0:01:53  \r",
      "    374,243,328  16%   24.18MB/s    0:01:15  \r",
      "    400,457,728  17%   23.62MB/s    0:01:16  \r",
      "    432,144,384  19%   25.52MB/s    0:01:09  \r",
      "    463,175,680  20%   27.95MB/s    0:01:02  \r",
      "    485,916,672  21%   26.10MB/s    0:01:05  \r",
      "    512,131,072  22%   26.10MB/s    0:01:04  \r",
      "    537,100,288  23%   24.53MB/s    0:01:08  \r",
      "    561,872,896  25%   23.54MB/s    0:01:09  \r",
      "    588,152,832  26%   24.38MB/s    0:01:06  \r",
      "    614,825,984  27%   24.49MB/s    0:01:05  \r",
      "    637,435,904  28%   23.93MB/s    0:01:05  \r",
      "    664,731,648  29%   24.53MB/s    0:01:02  \r",
      "    687,800,320  30%   23.77MB/s    0:01:04  \r",
      "    702,251,008  31%    9.00MB/s    0:02:47  \r",
      "    706,445,312  31%    6.49MB/s    0:03:51  \r",
      "    710,377,472  31%    3.47MB/s    0:07:12  \r",
      "    722,960,384  32%    2.67MB/s    0:09:16  \r",
      "    727,154,688  32%    3.25MB/s    0:07:37  \r",
      "    736,067,584  32%    4.37MB/s    0:05:37  \r",
      "    743,964,672  33%    7.21MB/s    0:03:23  \r",
      "    752,353,280  33%    6.13MB/s    0:03:58  \r",
      "    761,004,032  33%    6.96MB/s    0:03:28  \r",
      "    773,292,032  34%    7.12MB/s    0:03:22  \r",
      "    790,134,784  35%    9.55MB/s    0:02:29  \r",
      "    806,649,856  35%   11.59MB/s    0:02:01  \r",
      "    821,395,456  36%   13.19MB/s    0:01:45  \r",
      "    836,468,736  37%   15.07MB/s    0:01:31  \r",
      "    846,692,352  37%   13.29MB/s    0:01:42  \r",
      "    864,518,144  38%   13.54MB/s    0:01:39  \r",
      "    886,767,616  39%   15.30MB/s    0:01:26  \r",
      "    918,028,288  40%   19.08MB/s    0:01:08  \r",
      "    947,355,648  42%   23.76MB/s    0:00:53  \r",
      "    977,666,048  43%   26.83MB/s    0:00:46  \r",
      "    993,492,992  44%   25.27MB/s    0:00:48  \r",
      "  1,024,098,304  45%   25.13MB/s    0:00:47  \r",
      "  1,049,886,720  46%   24.43MB/s    0:00:47  \r",
      "  1,076,002,816  47%   23.43MB/s    0:00:48  \r",
      "  1,103,298,560  49%   26.19MB/s    0:00:42  \r",
      "  1,133,510,656  50%   26.09MB/s    0:00:41  \r",
      "  1,157,824,512  51%   25.75MB/s    0:00:41  \r",
      "  1,178,238,976  52%   24.39MB/s    0:00:42  \r",
      "  1,206,321,152  53%   24.57MB/s    0:00:41  \r",
      "  1,231,355,904  54%   23.33MB/s    0:00:42  \r",
      "  1,263,140,864  56%   25.12MB/s    0:00:38  \r",
      "  1,294,860,288  57%   27.81MB/s    0:00:33  \r",
      "  1,325,170,688  58%   28.35MB/s    0:00:31  \r",
      "  1,356,824,576  60%   29.94MB/s    0:00:29  \r",
      "  1,388,085,248  61%   29.81MB/s    0:00:28  \r",
      "  1,419,870,208  63%   29.82MB/s    0:00:27  \r",
      "  1,451,458,560  64%   30.12MB/s    0:00:25  \r",
      "  1,482,653,696  65%   30.02MB/s    0:00:24  \r",
      "  1,505,460,224  66%   27.83MB/s    0:00:26  \r",
      "  1,526,464,512  67%   25.28MB/s    0:00:27  \r",
      "  1,535,377,408  68%   15.81MB/s    0:00:43  \r",
      "  1,541,406,720  68%   10.21MB/s    0:01:07  \r",
      "  1,548,845,056  68%    7.57MB/s    0:01:30  \r",
      "  1,553,694,720  69%    4.72MB/s    0:02:23  \r",
      "  1,568,112,640  69%    6.70MB/s    0:01:38  \r",
      "  1,576,501,248  70%    7.83MB/s    0:01:23  \r",
      "  1,593,540,608  70%    9.17MB/s    0:01:09  \r",
      "  1,601,667,072  71%    9.23MB/s    0:01:08  \r",
      "  1,606,123,520  71%    6.59MB/s    0:01:34  \r",
      "  1,620,541,440  72%    7.62MB/s    0:01:20  \r",
      "  1,636,270,080  72%    7.93MB/s    0:01:15  \r",
      "  1,650,425,856  73%    9.66MB/s    0:01:00  \r",
      "  1,667,203,072  74%   14.28MB/s    0:00:39  \r",
      "  1,681,883,136  74%   14.32MB/s    0:00:38  \r",
      "  1,697,939,456  75%   14.39MB/s    0:00:37  \r",
      "  1,717,010,432  76%   14.98MB/s    0:00:34  \r",
      "  1,748,533,248  77%   18.31MB/s    0:00:26  \r",
      "  1,772,486,656  78%   20.72MB/s    0:00:22  \r",
      "  1,795,391,488  79%   22.16MB/s    0:00:19  \r",
      "  1,825,931,264  81%   25.84MB/s    0:00:15  \r",
      "  1,853,947,904  82%   25.00MB/s    0:00:15  \r",
      "  1,885,634,560  83%   26.84MB/s    0:00:13  \r",
      "  1,911,652,352  85%   27.73MB/s    0:00:11  \r",
      "  1,939,308,544  86%   27.03MB/s    0:00:11  \r",
      "  1,970,307,072  87%   27.75MB/s    0:00:09  \r",
      "  2,000,912,384  89%   27.49MB/s    0:00:08  \r",
      "  2,032,926,720  90%   28.92MB/s    0:00:07  \r",
      "  2,063,564,800  91%   29.64MB/s    0:00:06  \r",
      "  2,094,989,312  93%   29.74MB/s    0:00:04  \r",
      "  2,122,350,592  94%   28.97MB/s    0:00:04  \r",
      "  2,141,257,728  95%   25.84MB/s    0:00:04  \r",
      "  2,163,736,576  96%   20.18MB/s    0:00:04  \r",
      "  2,196,897,792  97%   20.53MB/s    0:00:02  \r",
      "  2,220,064,768  98%   19.69MB/s    0:00:01  \r",
      "  2,227,961,856  99%   15.44MB/s    0:00:01  \r",
      "  2,241,757,184  99%   16.11MB/s    0:00:00  \r",
      "  2,247,117,885 100%   17.87MB/s    0:01:59 (xfr#1, to-chk=0/1)\n",
      "\n",
      "sent 30 bytes  received 2,247,392,295 bytes  18,497,056.17 bytes/sec\n",
      "total size is 2,247,117,885  speedup is 1.00\n",
      "\n",
      "\n",
      "----------------------------------------------------------\n",
      "total 2.1G\n",
      "-rw-r--r-- 1 sam users 2.1G Feb 11 12:13 Pgenerosa_v070.fa\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cd ${wd}\n",
    "\n",
    "rsync \\\n",
    "--archive \\\n",
    "--verbose \\\n",
    "--progress \\\n",
    "owl:/volume1/web/halfshell/genomic-databank/${fasta} .\n",
    "\n",
    "echo \"\"\n",
    "echo \"\"\n",
    "echo \"----------------------------------------------------------\"\n",
    "\n",
    "ls -lh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### If need to download via wget, change cell below to code, instead of markdown\n",
    "\n",
    "#### Also, change cell with md5 checksum comparisons to code, instead of markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "%%bash\n",
    "time \\\n",
    "wget ${fasta_url} \\\n",
    "--quiet \\\n",
    "--directory-prefix=${wd}\n",
    "\n",
    "ls -lh ${wd}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Verify MD5 checksum\n",
    "\n",
    "Original MD5 checksum taken from GitHub Genomic Resource linked above.\n",
    "\n",
    "Use ```md5sum``` to generate checksum from downloaded FastA file and ```awk``` to print the first field (i.e. the checksum value). This is saved to the variable: ```dl_md5```\n",
    "\n",
    "Then, check for differences between the two variables. \n",
    "\n",
    "No output confirms no difference."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "%%bash\n",
    "md5=${checksum}\n",
    "dl_md5=$(md5sum ${wd}/${fasta} | awk '{ print $1 }')\n",
    "diff <(echo \"$md5\") <(echo \"$dl_md5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trim FastA header length to avoid RepeatMasker error of length >50 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original FastA header:\n",
      "\n",
      "---------------------------------\n",
      "Updated FastA header:\n",
      ">PGA_scaffold1\n",
      "total 2.1G\n",
      "-rw-rw-r-- 1 sam sam 2.1G Aug  6 07:48 Pgenerosa_v070.fa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "bash: line 3: head-n: command not found\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cd ${wd}\n",
    "echo \"Original FastA header:\"\n",
    "head -n 1 \"${fasta}\"\n",
    "\n",
    "echo \"\"\n",
    "\n",
    "cut -f 1-2 -d \"_\" \"${fasta}\" > Pgenerosa_v070.fa.tmp\n",
    "mv Pgenerosa_v070.fa.tmp \"${fasta}\"\n",
    "\n",
    "echo \"---------------------------------\"\n",
    "\n",
    "echo \"Updated FastA header:\"\n",
    "head -n 1 \"${fasta}\"\n",
    "\n",
    "ls -ltrh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run RepeatMasker with _all_ species setting and following options:\n",
    "\n",
    "```-species \"all\"``` : Sets species to all\n",
    "\n",
    "```-par ${cpus}``` : Use n CPU threads\n",
    "\n",
    "```-gff``` : Create GFF output file (in addition to default files)\n",
    "\n",
    "```-excln``` : Adjusts output table calculations to exclude sequence runs of >=25Ns. Useful for draft genome assemblies.\n",
    "\n",
    "```-1>``` : Send stdout to file instead of printing to notebook.\n",
    "\n",
    "```-2>``` : Send stderr to file instead of printing to notebook.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "real\t5729m38.010s\n",
      "user\t88723m33.224s\n",
      "sys\t786m15.300s\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "cd ${wd}\n",
    "time \\\n",
    "${repeat_masker} \\\n",
    "${fasta} \\\n",
    "-species \"all\" \\\n",
    "-par ${cpus} \\\n",
    "-gff \\\n",
    "-excln \\\n",
    "1> stdout.out \\\n",
    "2> stderr.err\n",
    "\n",
    "sed '/^Subject:/ s/ / repeatmasker_pgenv070_all JOB COMPLETE/' ~/.default-subject.mail | msmtp \"$EMAIL\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 5.1G\n",
      "-rw-rw-r-- 1 sam sam 2.1G Aug  6 07:48 Pgenerosa_v070.fa\n",
      "-rw-rw-r-- 1 sam sam 438M Aug 10 07:19 Pgenerosa_v070.fa.cat.gz\n",
      "-rw-rw-r-- 1 sam sam 2.2G Aug 10 07:18 Pgenerosa_v070.fa.masked\n",
      "-rw-rw-r-- 1 sam sam 254M Aug 10 07:19 Pgenerosa_v070.fa.out\n",
      "-rw-rw-r-- 1 sam sam 157M Aug 10 07:18 Pgenerosa_v070.fa.out.gff\n",
      "-rw-rw-r-- 1 sam sam 2.4K Aug 10 07:18 Pgenerosa_v070.fa.tbl\n",
      "-rw-rw-r-- 1 sam sam    0 Aug  6 07:49 stderr.err\n",
      "-rw-rw-r-- 1 sam sam 6.8M Aug 10 07:18 stdout.out\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "ls -lh ${wd}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SUMMARY TABLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "file name: Pgenerosa_v070.fa        \n",
      "sequences:        313649\n",
      "total length: 2205688688 bp  (2005531528 bp excl N/X-runs)\n",
      "GC level:         33.92 %\n",
      "bases masked:  175175579 bp ( 8.73 %)\n",
      "==================================================\n",
      "               number of      length   percentage\n",
      "               elements*    occupied  of sequence\n",
      "--------------------------------------------------\n",
      "Retroelements       565711     87788537 bp    4.38 %\n",
      "   SINEs:           332333     39506023 bp    1.97 %\n",
      "   Penelope           6883       788411 bp    0.04 %\n",
      "   LINEs:           142649     32744907 bp    1.63 %\n",
      "    CRE/SLACS         1237       100944 bp    0.01 %\n",
      "     L2/CR1/Rex      40317      7764197 bp    0.39 %\n",
      "     R1/LOA/Jockey   10137      2942539 bp    0.15 %\n",
      "     R2/R4/NeSL       3825       551996 bp    0.03 %\n",
      "     RTE/Bov-B       26939      6768723 bp    0.34 %\n",
      "     L1/CIN4         21435      4046589 bp    0.20 %\n",
      "   LTR elements:     90729     15537607 bp    0.77 %\n",
      "     BEL/Pao          6594       918331 bp    0.05 %\n",
      "     Ty1/Copia       16409      1268565 bp    0.06 %\n",
      "     Gypsy/DIRS1     50972     11376086 bp    0.57 %\n",
      "       Retroviral     9680       690936 bp    0.03 %\n",
      "\n",
      "DNA transposons     259955     34987123 bp    1.74 %\n",
      "   hobo-Activator    29756      3192075 bp    0.16 %\n",
      "   Tc1-IS630-Pogo    67456      9717356 bp    0.48 %\n",
      "   En-Spm                0            0 bp    0.00 %\n",
      "   MuDR-IS905            0            0 bp    0.00 %\n",
      "   PiggyBac           1553       121136 bp    0.01 %\n",
      "   Tourist/Harbinger  7596      1054167 bp    0.05 %\n",
      "   Other (Mirage,     1803       123196 bp    0.01 %\n",
      "    P-element, Transib)\n",
      "\n",
      "Rolling-circles          0            0 bp    0.00 %\n",
      "\n",
      "Unclassified:        99928     13654973 bp    0.68 %\n",
      "\n",
      "Total interspersed repeats:   136430633 bp    6.80 %\n",
      "\n",
      "\n",
      "Small RNA:           42601      2192413 bp    0.11 %\n",
      "\n",
      "Satellites:          33350      6282246 bp    0.31 %\n",
      "Simple repeats:     596607     32793030 bp    1.64 %\n",
      "Low complexity:      75831      3754962 bp    0.19 %\n",
      "==================================================\n",
      "\n",
      "* most repeats fragmented by insertions or deletions\n",
      "  have been counted as one element\n",
      "  Runs of >=20 X/Ns in query were excluded in % calcs\n",
      "\n",
      "\n",
      "The query species was assumed to be root          \n",
      "RepeatMasker Combined Database: Dfam_Consensus-20170127, RepBase-20170127\n",
      "        \n",
      "run with rmblastn version 2.6.0+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cat ${wd}/${fasta}.tbl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Delete FastA (not needed) and `rsync` to my folder on Gannet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sending incremental file list\n",
      "./\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/Pgenerosa_v070.fa.cat.gz\n",
      "\r",
      "         32,768   0%    0.00kB/s    0:00:00  \r",
      "    110,788,608  24%  105.10MB/s    0:00:03  \r",
      "    172,490,752  37%   70.17MB/s    0:00:03  \r",
      "    284,721,152  62%   80.92MB/s    0:00:02  \r",
      "    358,449,152  78%   78.45MB/s    0:00:01  \r",
      "    446,464,000  97%   73.27MB/s    0:00:00  \r",
      "    459,015,612 100%   79.27MB/s    0:00:05 (xfr#1, to-chk=6/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/Pgenerosa_v070.fa.masked\n",
      "\r",
      "         32,768   0%  217.69kB/s    2:52:43  \r",
      "     72,941,568   3%   69.35MB/s    0:00:30  \r",
      "    118,947,840   5%   55.01MB/s    0:00:37  \r",
      "    171,802,624   7%   53.53MB/s    0:00:38  \r",
      "    269,680,640  11%   63.35MB/s    0:00:30  \r",
      "    339,345,408  15%   62.29MB/s    0:00:30  \r",
      "    413,728,768  18%   69.95MB/s    0:00:25  \r",
      "    506,986,496  22%   79.54MB/s    0:00:21  \r",
      "    579,960,832  25%   73.63MB/s    0:00:22  \r",
      "    675,840,000  29%   80.29MB/s    0:00:19  \r",
      "    764,805,120  33%   83.77MB/s    0:00:17  \r",
      "    852,787,200  37%   82.30MB/s    0:00:16  \r",
      "    937,852,928  41%   82.74MB/s    0:00:15  \r",
      "  1,046,183,936  46%   85.62MB/s    0:00:13  \r",
      "  1,142,882,304  50%   87.41MB/s    0:00:12  \r",
      "  1,243,643,904  55%   90.63MB/s    0:00:10  \r",
      "  1,341,063,168  59%   96.20MB/s    0:00:09  \r",
      "  1,437,761,536  63%   93.41MB/s    0:00:08  \r",
      "  1,537,409,024  68%   94.11MB/s    0:00:07  \r",
      "  1,631,879,168  72%   92.63MB/s    0:00:06  \r",
      "  1,726,087,168  76%   91.87MB/s    0:00:05  \r",
      "  1,823,735,808  80%   92.09MB/s    0:00:04  \r",
      "  1,909,686,272  84%   88.49MB/s    0:00:03  \r",
      "  2,012,315,648  89%   90.41MB/s    0:00:02  \r",
      "  2,117,206,016  93%   92.97MB/s    0:00:01  \r",
      "  2,219,671,552  98%   94.12MB/s    0:00:00  \r",
      "  2,256,119,169 100%   84.20MB/s    0:00:25 (xfr#2, to-chk=5/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/Pgenerosa_v070.fa.out\n",
      "\r",
      "         32,768   0%   93.84kB/s    0:47:12  \r",
      "     70,025,216  26%   66.78MB/s    0:00:02  \r",
      "    164,691,968  61%   78.02MB/s    0:00:01  \r",
      "    265,847,056 100%   84.79MB/s    0:00:02 (xfr#3, to-chk=4/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/Pgenerosa_v070.fa.out.gff\n",
      "\r",
      "         32,768   0%   32.79kB/s    1:23:34  \r",
      "        688,128   0%  669.32kB/s    0:04:04  \r",
      "    101,646,336  61%   48.23MB/s    0:00:01  \r",
      "    164,437,744 100%   56.43MB/s    0:00:02 (xfr#4, to-chk=3/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/Pgenerosa_v070.fa.tbl\n",
      "\r",
      "          2,423 100%    3.08kB/s    0:00:00  \r",
      "          2,423 100%    3.08kB/s    0:00:00 (xfr#5, to-chk=2/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/stderr.err\n",
      "\r",
      "              0 100%    0.00kB/s    0:00:00 (xfr#6, to-chk=1/9)\n",
      "20190805_Pgenerosa_v070_repeatmasker_all/stdout.out\n",
      "\r",
      "         32,768   0%   37.21kB/s    0:03:10  \r",
      "      6,750,208  94%    6.44MB/s    0:00:00  \r",
      "      7,126,317 100%    6.79MB/s    0:00:01 (xfr#7, to-chk=0/9)\n",
      "\n",
      "sent 3,153,318,561 bytes  received 163 bytes  86,392,293.81 bytes/sec\n",
      "total size is 3,152,548,321  speedup is 1.00\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "cd ${wd}\n",
    "rm ${fasta}\n",
    "cd /home/sam/analyses/\n",
    "\n",
    "rsync \\\n",
    "--archive \\\n",
    "--verbose \\\n",
    "--progress \\\n",
    "--relative \\\n",
    "./20190805_Pgenerosa_v070_repeatmasker_all \\\n",
    "gannet:/volume2/web/Atumefaciens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}