{ "cells": [ { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "%%html\n", "" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%html\n", "" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%html\n", "" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\"Dask\n", " \n", "\n", "# JupyterLab on Qarnot\n", "\n", "## Add your Qarnot token" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import io\n", "import pandas as pd\n", "import ipywidgets as widgets\n", "from tkinter import Tk, filedialog\n", "from IPython.display import clear_output, display, HTML" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "token = widgets.Password(\n", " placeholder='Enter token',\n", " description='Qarnot token:',\n", " disabled=False,\n", " layout=widgets.Layout(width='40%')\n", ")\n", "display(token)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Add your public SSH key" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ssh_key = widgets.Text(\n", " placeholder='Enter Key',\n", " description='SSH Public Key:',\n", " disabled=False,\n", " style={'description_width': 'initial'},\n", " layout=widgets.Layout(width='40%')\n", ")\n", "display(ssh_key)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Task and bucket names" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from ipywidgets import Layout, Box, Label, Text\n", "\n", "form_item_layout = Layout(\n", " display='flex',\n", " flex_flow='row',\n", " justify_content='space-between'\n", ")\n", "\n", "form_items = [\n", " Box([Label(value='Task name:'),\n", " Text(value='jupyterlab', placeholder='Enter task name')], layout=form_item_layout),\n", " Box([Label(value='Input bucket name:'),\n", " Text(value='jupyterlab-in', placeholder='Enter input bucket name')], layout=form_item_layout),\n", " Box([Label(value='Output bucket name:'),\n", " Text(value='jupyterlab-out', placeholder='Enter input bucket name')], layout=form_item_layout),\n", "]\n", "\n", "form = Box(form_items, layout=Layout(\n", " display='flex',\n", " flex_flow='column',\n", " align_items='stretch',\n", " width='50%'\n", "))\n", "\n", "form" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Select the software you want\n", "\n", "Select one of the available JupyterLab images that come pre-installed with popular Machine Learning softwares" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "soft_options = [\n", " ('Base','qarnotlab/jupyterlab-base'), ('Scikit-learn', 'qarnotlab/jupyterlab-sklearn'), \n", " ('Tensorflow', 'qarnotlab/jupyterlab-tensorflow-cpu'), ('Pytorch', 'qarnotlab/jupyterlab-pytorch-cpu')\n", "]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "soft = widgets.Select(\n", " options=soft_options,\n", " value='qarnotlab/jupyterlab-base',\n", " rows=4,\n", " description='JupyterLab Software:',\n", " disabled=False,\n", " style={'description_width': 'initial'},\n", " layout=widgets.Layout(width='40%')\n", ")\n", "display(soft)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Upload your data to binder " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "file = widgets.FileUpload(\n", " multiple=True # True to accept multiple files upload else False\n", ")\n", "display(file)\n", "\n", "def on_upload_change(change):\n", " # create input folder\n", " ! mkdir -p input_binder/\n", " \n", " for i, df in enumerate(file.value):\n", " content = file.value[df]['content']\n", " with open('input_binder/'+df, 'wb') as f: f.write(content)\n", "\n", "file.observe(on_upload_change, names='_counter')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Use previous output bucket as input" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "prev_out_bucket = widgets.Text(\n", " value='jupyterlab-out',\n", " description='Output bucket to use:',\n", " disabled=False,\n", " style={'description_width': 'initial'},\n", " layout=widgets.Layout(width='40%')\n", ")\n", "\n", "check = widgets.Checkbox(\n", " value=False,\n", " description='Check to use previous output bucket as input',\n", " disabled=False,\n", " indent=False\n", ")\n", "\n", "display(prev_out_bucket, check)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Launch JupyterLab on Qarnot" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "from run_qarnot import submit_task\n", "from ipywidgets import Button, Output, Layout\n", "from traitlets import traitlets\n", "\n", "class LoadedButton(widgets.Button):\n", " \"\"\"A button that can holds a value as a attribute.\"\"\"\n", "\n", " def __init__(self, value=None, *args, **kwargs):\n", " super(LoadedButton, self).__init__(*args, **kwargs)\n", " # Create the value attribute.\n", " self.add_traits(value=traitlets.Any(value))\n", "\n", "button = LoadedButton(description=\"Start JupyterLab on Qarnot\", layout=Layout(width='auto'))\n", "output = Output()\n", "display(button, output)\n", "\n", "def on_button_clicked(b):\n", " \n", " # Dictionary for data storage\n", " param_dict = {'token':'', 'ssh_key':'', 'task':'', 'in_bucket':'', 'out_bucket':'',\n", " 'docker_repo':'', 'prev_out_bucket':'', 'use_output_bucket':''}\n", " param_list = list(param_dict)\n", " \n", " output.clear_output()\n", " with output:\n", " try:\n", " param_dict['token'] = token.value\n", " param_dict['ssh_key'] = ssh_key.value\n", " # Retrieve data from form in dictionary\n", " for _, (key, elem) in enumerate(zip(param_list[2:5], form.children)):\n", " param_dict[key] = str(elem.children[1].value)\n", " param_dict['soft'] = soft.value\n", " param_dict['prev_out_bucket'] = prev_out_bucket.value\n", " param_dict['use_output_bucket'] = check.value\n", " except IndexError:\n", " print(\"Some fields were not properly filled\")\n", " if param_dict['ssh_key'] == '' or param_dict['token'] == '':\n", " return(print('Some fields were not properly filled'))\n", " \n", " # Launch computation\n", " ! mkdir -p logs/\n", " # Launch task and get port, link and uuid\n", " forward_port, link, uuid = submit_task(param_dict)\n", " b.value = uuid\n", " ssh_cmd = \"ssh -L 8888:localhost:8888 -o StrictHostKeyChecking=no root@forward01.qarnot.net -p \"+str(forward_port)\n", " \n", " # Command to establish ssh connection\n", " ssh_prompt = widgets.HTML(\n", " value=f\"{'Copy and paste the following command in your terminal to connect via SSH with Qarnot'}\"\n", " )\n", " \n", " # Generate link to JupyterLab instance (with secret token)\n", " html_link = \"JupyterLab\"\n", " text = widgets.HTML(\n", " value = f\"{'Click on the link below to get access to your JupyterLab instance:'}\"\n", " )\n", " hyper_link = widgets.HTML(value=html_link,)\n", " \n", " # Display\n", " display(ssh_prompt)\n", " print(ssh_cmd)\n", " display(text)\n", " display(hyper_link)\n", " \n", "button.on_click(on_button_clicked)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get results and abort task" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from run_qarnot import abort_task\n", "\n", "button_abort = Button(description=\"Abort Task\", layout=Layout(width='auto'))\n", "output_abort = Output()\n", "display(button_abort, output_abort)\n", "\n", "def on_button_clicked_abort(b):\n", " \n", " # Launch computation\n", " output_abort.clear_output()\n", " with output_abort:\n", " print('Aborting Task...')\n", " abort_task(token.value, button.value)\n", "\n", "button_abort.on_click(on_button_clicked_abort)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Download results locally" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from shutil import make_archive\n", "from IPython.display import FileLink\n", "\n", "download_button = Button(description=\"Download outputs\", layout=Layout(width='auto'))\n", "download_output = Output()\n", "display(download_button, download_output)\n", "\n", "def on_download_button_clicked(b):\n", " download_output.clear_output()\n", " with download_output:\n", " try:\n", " print('Compressing outputs into .zip file...')\n", " make_archive('output', 'zip', 'outputs_binder/', verbose = 10)\n", " link = FileLink(\n", " path='output.zip', \n", " result_html_prefix='Your output .zip file is ready ! \\\n", " Click the following link to download it: '\n", " )\n", " display(link)\n", " \n", " except FileNotFoundError:\n", " print(\"Output files not available\")\n", " \n", "download_button.on_click(on_download_button_clicked)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }