{
"cells": [
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%html\n",
""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%html\n",
""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%html\n",
""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
" \n",
"\n",
"# JupyterLab on Qarnot\n",
"\n",
"## Add your Qarnot token"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import io\n",
"import pandas as pd\n",
"import ipywidgets as widgets\n",
"from tkinter import Tk, filedialog\n",
"from IPython.display import clear_output, display, HTML"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"token = widgets.Password(\n",
" placeholder='Enter token',\n",
" description='Qarnot token:',\n",
" disabled=False,\n",
" layout=widgets.Layout(width='40%')\n",
")\n",
"display(token)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Add your public SSH key"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ssh_key = widgets.Text(\n",
" placeholder='Enter Key',\n",
" description='SSH Public Key:',\n",
" disabled=False,\n",
" style={'description_width': 'initial'},\n",
" layout=widgets.Layout(width='40%')\n",
")\n",
"display(ssh_key)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Task and bucket names"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from ipywidgets import Layout, Box, Label, Text\n",
"\n",
"form_item_layout = Layout(\n",
" display='flex',\n",
" flex_flow='row',\n",
" justify_content='space-between'\n",
")\n",
"\n",
"form_items = [\n",
" Box([Label(value='Task name:'),\n",
" Text(value='jupyterlab', placeholder='Enter task name')], layout=form_item_layout),\n",
" Box([Label(value='Input bucket name:'),\n",
" Text(value='jupyterlab-in', placeholder='Enter input bucket name')], layout=form_item_layout),\n",
" Box([Label(value='Output bucket name:'),\n",
" Text(value='jupyterlab-out', placeholder='Enter input bucket name')], layout=form_item_layout),\n",
"]\n",
"\n",
"form = Box(form_items, layout=Layout(\n",
" display='flex',\n",
" flex_flow='column',\n",
" align_items='stretch',\n",
" width='50%'\n",
"))\n",
"\n",
"form"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Select the software you want\n",
"\n",
"Select one of the available JupyterLab images that come pre-installed with popular Machine Learning softwares"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"soft_options = [\n",
" ('Base','qarnotlab/jupyterlab-base'), ('Scikit-learn', 'qarnotlab/jupyterlab-sklearn'), \n",
" ('Tensorflow', 'qarnotlab/jupyterlab-tensorflow-cpu'), ('Pytorch', 'qarnotlab/jupyterlab-pytorch-cpu')\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"soft = widgets.Select(\n",
" options=soft_options,\n",
" value='qarnotlab/jupyterlab-base',\n",
" rows=4,\n",
" description='JupyterLab Software:',\n",
" disabled=False,\n",
" style={'description_width': 'initial'},\n",
" layout=widgets.Layout(width='40%')\n",
")\n",
"display(soft)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Upload your data to binder "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"file = widgets.FileUpload(\n",
" multiple=True # True to accept multiple files upload else False\n",
")\n",
"display(file)\n",
"\n",
"def on_upload_change(change):\n",
" # create input folder\n",
" ! mkdir -p input_binder/\n",
" \n",
" for i, df in enumerate(file.value):\n",
" content = file.value[df]['content']\n",
" with open('input_binder/'+df, 'wb') as f: f.write(content)\n",
"\n",
"file.observe(on_upload_change, names='_counter')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Use previous output bucket as input"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"prev_out_bucket = widgets.Text(\n",
" value='jupyterlab-out',\n",
" description='Output bucket to use:',\n",
" disabled=False,\n",
" style={'description_width': 'initial'},\n",
" layout=widgets.Layout(width='40%')\n",
")\n",
"\n",
"check = widgets.Checkbox(\n",
" value=False,\n",
" description='Check to use previous output bucket as input',\n",
" disabled=False,\n",
" indent=False\n",
")\n",
"\n",
"display(prev_out_bucket, check)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Launch JupyterLab on Qarnot"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from run_qarnot import submit_task\n",
"from ipywidgets import Button, Output, Layout\n",
"from traitlets import traitlets\n",
"\n",
"class LoadedButton(widgets.Button):\n",
" \"\"\"A button that can holds a value as a attribute.\"\"\"\n",
"\n",
" def __init__(self, value=None, *args, **kwargs):\n",
" super(LoadedButton, self).__init__(*args, **kwargs)\n",
" # Create the value attribute.\n",
" self.add_traits(value=traitlets.Any(value))\n",
"\n",
"button = LoadedButton(description=\"Start JupyterLab on Qarnot\", layout=Layout(width='auto'))\n",
"output = Output()\n",
"display(button, output)\n",
"\n",
"def on_button_clicked(b):\n",
" \n",
" # Dictionary for data storage\n",
" param_dict = {'token':'', 'ssh_key':'', 'task':'', 'in_bucket':'', 'out_bucket':'',\n",
" 'docker_repo':'', 'prev_out_bucket':'', 'use_output_bucket':''}\n",
" param_list = list(param_dict)\n",
" \n",
" output.clear_output()\n",
" with output:\n",
" try:\n",
" param_dict['token'] = token.value\n",
" param_dict['ssh_key'] = ssh_key.value\n",
" # Retrieve data from form in dictionary\n",
" for _, (key, elem) in enumerate(zip(param_list[2:5], form.children)):\n",
" param_dict[key] = str(elem.children[1].value)\n",
" param_dict['soft'] = soft.value\n",
" param_dict['prev_out_bucket'] = prev_out_bucket.value\n",
" param_dict['use_output_bucket'] = check.value\n",
" except IndexError:\n",
" print(\"Some fields were not properly filled\")\n",
" if param_dict['ssh_key'] == '' or param_dict['token'] == '':\n",
" return(print('Some fields were not properly filled'))\n",
" \n",
" # Launch computation\n",
" ! mkdir -p logs/\n",
" # Launch task and get port, link and uuid\n",
" forward_port, link, uuid = submit_task(param_dict)\n",
" b.value = uuid\n",
" ssh_cmd = \"ssh -L 8888:localhost:8888 -o StrictHostKeyChecking=no root@forward01.qarnot.net -p \"+str(forward_port)\n",
" \n",
" # Command to establish ssh connection\n",
" ssh_prompt = widgets.HTML(\n",
" value=f\"{'Copy and paste the following command in your terminal to connect via SSH with Qarnot'}\"\n",
" )\n",
" \n",
" # Generate link to JupyterLab instance (with secret token)\n",
" html_link = \"JupyterLab\"\n",
" text = widgets.HTML(\n",
" value = f\"{'Click on the link below to get access to your JupyterLab instance:'}\"\n",
" )\n",
" hyper_link = widgets.HTML(value=html_link,)\n",
" \n",
" # Display\n",
" display(ssh_prompt)\n",
" print(ssh_cmd)\n",
" display(text)\n",
" display(hyper_link)\n",
" \n",
"button.on_click(on_button_clicked)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get results and abort task"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from run_qarnot import abort_task\n",
"\n",
"button_abort = Button(description=\"Abort Task\", layout=Layout(width='auto'))\n",
"output_abort = Output()\n",
"display(button_abort, output_abort)\n",
"\n",
"def on_button_clicked_abort(b):\n",
" \n",
" # Launch computation\n",
" output_abort.clear_output()\n",
" with output_abort:\n",
" print('Aborting Task...')\n",
" abort_task(token.value, button.value)\n",
"\n",
"button_abort.on_click(on_button_clicked_abort)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Download results locally"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from shutil import make_archive\n",
"from IPython.display import FileLink\n",
"\n",
"download_button = Button(description=\"Download outputs\", layout=Layout(width='auto'))\n",
"download_output = Output()\n",
"display(download_button, download_output)\n",
"\n",
"def on_download_button_clicked(b):\n",
" download_output.clear_output()\n",
" with download_output:\n",
" try:\n",
" print('Compressing outputs into .zip file...')\n",
" make_archive('output', 'zip', 'outputs_binder/', verbose = 10)\n",
" link = FileLink(\n",
" path='output.zip', \n",
" result_html_prefix='Your output .zip file is ready ! \\\n",
" Click the following link to download it: '\n",
" )\n",
" display(link)\n",
" \n",
" except FileNotFoundError:\n",
" print(\"Output files not available\")\n",
" \n",
"download_button.on_click(on_download_button_clicked)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}