{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "import boto3\n", "import botocore\n", "import paramiko" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "mergedMatricesDir='/nrnb/users/btsui/Data/all_seq/rnaseq_merged/'\n", "inFnameS=pd.Series(os.listdir(mergedMatricesDir))" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "include_m1=(inFnameS.str.contains('gene_symbol')&(~inFnameS.str.contains('.gz')))\n", "include_m2=inFnameS.isin(['merged_kallisto_run_info.pickle'])\n", "transferFnames=inFnameS[include_m1|include_m2]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "key = paramiko.RSAKey.from_private_key_file('/cellar/users/btsui/.ssh/jupyter_hub.pem')\n", "client = paramiko.SSHClient()\n", "client.set_missing_host_key_policy(paramiko.AutoAddPolicy())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\n", "* [Launch the T3 micro instance on AWS with k8s.ebs_20g.efs_automount (ami-04052dc36ee4ac5c1)](https://us-west-2.console.aws.amazon.com/ec2/v2/home?region=us-west-2#Instances:sort=launchTime)\n", "* Modify the instance_ip to the AWS for data uploading instance" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "instance_ip='ec2-34-215-138-8.us-west-2.compute.amazonaws.com'\n", "mountEfs='sudo mount -t efs fs-e1636448:/ efs'\n", "\n", "#try:\n", "# Here 'ubuntu' is user name and 'instance_ip' is public IP of EC2\n", "client.connect(hostname=instance_ip, username=\"ec2-user\", pkey=key)\n", "# Execute a command(cmd) after connecting/ssh to an instance\n", "cmd='mkdir ~/efs'\n", "stdin, stdout, stderr = client.exec_command(cmd)\n", "cmd='sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 fs-e1636448.efs.us-west-2.amazonaws.com:/ ~/efs'\n", "stdin, stdout, stderr = client.exec_command(cmd)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b'' b''\n" ] } ], "source": [ "print(stdout.read(),stderr.read())" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "remoteDir='~/efs/all_seq/rnaseq_merged/'" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b''\n", "b'mkdir: cannot create directory \\xe2\\x80\\x98/home/ec2-user/efs/all_seq/rnaseq_merged/\\xe2\\x80\\x99: File exists\\n'\n" ] } ], "source": [ "cmd='mkdir '+remoteDir\n", "stdin, stdout, stderr = client.exec_command(cmd)\n", "print (stdout.read())\n", "print (stderr.read())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "addHostVerificationCmd=\"ssh -o StrictHostKeyChecking=no -i $HOME/.ssh/jupyter_hub.pem ec2-user@{instance_ip} ls\".format(instance_ip=instance_ip)\n", "os.system(addHostVerificationCmd)\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "myRsyncDirFormat='rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" {localFDir} ec2-user@{instance_ip}:{remoteDir}'" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Danio_rerio.gene_symbol.tpm.columns.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Danio_rerio.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.tpm.index.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.est_counts.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.tpm.columns.txt ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/merged_kallisto_run_info.pickle ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n", "0\n", "rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Mus_musculus.gene_symbol.est_counts.npy ec2-user@ec2-34-215-138-8.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/.\n" ] } ], "source": [ "for localFname in transferFnames.values:\n", " localFDir= (mergedMatricesDir+localFname)\n", " rsyncCmd=myRsyncDirFormat.format(localFDir=localFDir,instance_ip=instance_ip,remoteDir=remoteDir+'.')\n", " print (rsyncCmd)\n", " \n", " print (os.system(rsyncCmd))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "#!rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt ec2-user@ec2-52-13-69-172.us-west-2.compute.amazonaws.com:~/efs/all_seq/rnaseq_merged/." ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "#!rsync -Pvu -e \"ssh -i $HOME/.ssh/jupyter_hub.pem\" /nrnb/users/btsui/Data/all_seq/rnaseq_merged/Canis_familiaris.gene_symbol.tpm.index.txt \\\n", "#ec2-user@ec2-52-24-55-248.us-west-2.compute.amazonaws.com:~/." ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "authorized_keys id_dsa.pub\t id_rsa.jupyterhub.pub known_hosts~\r\n", "aws_upload.pem\t id_rsa\t\t id_rsa.pub\r\n", "id_dsa\t\t id_rsa.jupyterhub known_hosts\r\n" ] } ], "source": [ "#!ls $HOME/.ssh/" ] }, { "cell_type": "code", "execution_count": 105, "metadata": {}, "outputs": [], "source": [ "import seaborn" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "#!cp ./upload_AWS.ipynb ./../../Update_SRA_meta_data/." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }