{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
      "  from collections import Callable\n",
      "\n",
      "    You are using PySparkling of version 2.4.10, but your PySpark is of\n",
      "    version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n"
     ]
    }
   ],
   "source": [
    "from optimus.livy import Livy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "HOST = 'http://46.101.172.155:8998'\n",
    "livy = Livy(HOST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 52, 'name': None, 'appId': None, 'owner': None, 'proxyUser': None, 'state': 'starting', 'kind': 'pyspark', 'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None}, 'log': ['stdout: ', '\\nstderr: ']}\n"
     ]
    }
   ],
   "source": [
    "# Create session\n",
    "livy.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 52,\n",
       " 'name': None,\n",
       " 'appId': None,\n",
       " 'owner': None,\n",
       " 'proxyUser': None,\n",
       " 'state': 'starting',\n",
       " 'kind': 'pyspark',\n",
       " 'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None},\n",
       " 'log': ['stdout: ', '\\nstderr: ']}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get session info\n",
    "livy.session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 2, 'code': '1 + 1', 'state': 'available', 'output': {'status': 'ok', 'execution_count': 2, 'data': {'text/plain': '2'}}, 'progress': 1.0}\n"
     ]
    }
   ],
   "source": [
    "data = {'code': '1 + 1'}\n",
    "\n",
    "response = livy.execute(data)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 33, 'state': 'running', 'output': None, 'progress': 0.0}\n"
     ]
    }
   ],
   "source": [
    "import textwrap, pprint\n",
    "code  = \"\"\"\n",
    "    from optimus import Optimus    \n",
    "    op = Optimus(spark)\n",
    "    df = op.load.csv(\"https://raw.githubusercontent.com/ironmussa/Optimus/master/examples/data/foo.csv\")\n",
    "    \"\"\"\n",
    "\n",
    "response = livy.submit(code)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 34, 'state': 'waiting', 'output': None, 'progress': 0.0}\n"
     ]
    }
   ],
   "source": [
    "import textwrap, pprint\n",
    "code  = \"\"\"   \n",
    "    import json\n",
    "    json.dumps(df.to_json())\n",
    "    \"\"\"\n",
    "\n",
    "response = livy.submit(code)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = livy.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'[{\"id\": 1, \"product\": \"Cake\", \"firstName\": \"Luis\", \"price\": 10, \"lastName\": \"Alvarez$$%!\", \"billingId\": 123, \"dummyCol\": \"never\", \"birth\": \"1980/07/07\"}, {\"id\": 2, \"product\": \"piza\", \"firstName\": \"Andr\\\\u00e9\", \"price\": 8, \"lastName\": \"Amp\\\\u00e8re\", \"billingId\": 423, \"dummyCol\": \"gonna\", \"birth\": \"1950/07/08\"}, {\"id\": 3, \"product\": \"pizza\", \"firstName\": \"NiELS\", \"price\": 8, \"lastName\": \"B\\\\u00f6hr//((%%\", \"billingId\": 551, \"dummyCol\": \"give\", \"birth\": \"1990/07/09\"}, {\"id\": 4, \"product\": \"pizza\", \"firstName\": \"PAUL\", \"price\": 8, \"lastName\": \"dirac$\", \"billingId\": 521, \"dummyCol\": \"you\", \"birth\": \"1954/07/10\"}, {\"id\": 5, \"product\": \"pizza\", \"firstName\": \"Albert\", \"price\": 8, \"lastName\": \"Einstein\", \"billingId\": 634, \"dummyCol\": \"up\", \"birth\": \"1990/07/11\"}, {\"id\": 6, \"product\": \"arepa\", \"firstName\": \"Galileo\", \"price\": 5, \"lastName\": \"             GALiLEI\", \"billingId\": 672, \"dummyCol\": \"never\", \"birth\": \"1930/08/12\"}, {\"id\": 7, \"product\": \"taco\", \"firstName\": \"CaRL\", \"price\": 3, \"lastName\": \"Ga%%%uss\", \"billingId\": 323, \"dummyCol\": \"gonna\", \"birth\": \"1970/07/13\"}, {\"id\": 8, \"product\": \"taaaccoo\", \"firstName\": \"David\", \"price\": 3, \"lastName\": \"H$$$ilbert\", \"billingId\": 624, \"dummyCol\": \"let\", \"birth\": \"1950/07/14\"}, {\"id\": 9, \"product\": \"taco\", \"firstName\": \"Johannes\", \"price\": 3, \"lastName\": \"KEPLER\", \"billingId\": 735, \"dummyCol\": \"you\", \"birth\": \"1920/04/22\"}, {\"id\": 10, \"product\": \"taco\", \"firstName\": \"JaMES\", \"price\": 3, \"lastName\": \"M$$ax%%well\", \"billingId\": 875, \"dummyCol\": \"down\", \"birth\": \"1923/03/12\"}, {\"id\": 11, \"product\": \"pasta\", \"firstName\": \"Isaac\", \"price\": 9, \"lastName\": \"Newton\", \"billingId\": 992, \"dummyCol\": \"never \", \"birth\": \"1999/02/15\"}, {\"id\": 12, \"product\": \"pasta\", \"firstName\": \"Emmy%%\", \"price\": 9, \"lastName\": \"N\\\\u00f6ether$\", \"billingId\": 234, \"dummyCol\": \"gonna\", \"birth\": \"1993/12/08\"}, {\"id\": 13, \"product\": \"hamburguer\", \"firstName\": \"Max!!!\", \"price\": 4, \"lastName\": \"Planck!!!\", \"billingId\": 111, \"dummyCol\": \"run \", \"birth\": \"1994/01/04\"}, {\"id\": 14, \"product\": \"pizzza\", \"firstName\": \"Fred\", \"price\": 8, \"lastName\": \"Hoy&&&le\", \"billingId\": 553, \"dummyCol\": \"around\", \"birth\": \"1997/06/27\"}, {\"id\": 15, \"product\": \"pizza\", \"firstName\": \"(((   Heinrich )))))\", \"price\": 8, \"lastName\": \"Hertz\", \"billingId\": 116, \"dummyCol\": \"and\", \"birth\": \"1956/11/30\"}, {\"id\": 16, \"product\": \"BEER\", \"firstName\": \"William\", \"price\": 2, \"lastName\": \"Gilbert###\", \"billingId\": 886, \"dummyCol\": \"desert\", \"birth\": \"1958/03/26\"}, {\"id\": 17, \"product\": \"Rice\", \"firstName\": \"Marie\", \"price\": 1, \"lastName\": \"CURIE\", \"billingId\": 912, \"dummyCol\": \"you\", \"birth\": \"2000/03/22\"}, {\"id\": 18, \"product\": \"110790\", \"firstName\": \"Arthur\", \"price\": 5, \"lastName\": \"COM%%%pton\", \"billingId\": 812, \"dummyCol\": \"#\", \"birth\": \"1899/01/01\"}, {\"id\": 19, \"product\": \"null\", \"firstName\": \"JAMES\", \"price\": 10, \"lastName\": \"Chadwick\", \"billingId\": 467, \"dummyCol\": \"#\", \"birth\": \"1921/05/03\"}]'\n"
     ]
    },
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-96-0785c2deabf7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    346\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 348\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    349\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    350\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    335\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    336\u001b[0m         \"\"\"\n\u001b[1;32m--> 337\u001b[1;33m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    338\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    339\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m    353\u001b[0m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    354\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 355\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    356\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(a)\n",
    "json.loads(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://46.101.172.155:8998/sessions/46\n"
     ]
    }
   ],
   "source": [
    "livy.delete_session(46)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 52,\n",
       "  'name': None,\n",
       "  'appId': None,\n",
       "  'owner': None,\n",
       "  'proxyUser': None,\n",
       "  'state': 'idle',\n",
       "  'kind': 'pyspark',\n",
       "  'appInfo': {'driverLogUrl': None, 'sparkUiUrl': None},\n",
       "  'log': ['19/06/04 03:52:01 INFO Executor: Fetching spark://10.19.0.5:42304/jars/livy-repl_2.11-0.6.0-incubating.jar with timestamp 1559619584198',\n",
       "   '19/06/04 03:52:01 INFO Utils: Fetching spark://10.19.0.5:42304/jars/livy-repl_2.11-0.6.0-incubating.jar to /tmp/spark-e906068a-b29b-4594-aef1-58a2d37bde15/userFiles-d5869831-80e5-4da1-aa9c-37245d12c7f2/fetchFileTemp3930129784218452512.tmp',\n",
       "   '19/06/04 03:52:01 INFO Executor: Adding file:/tmp/spark-e906068a-b29b-4594-aef1-58a2d37bde15/userFiles-d5869831-80e5-4da1-aa9c-37245d12c7f2/livy-repl_2.11-0.6.0-incubating.jar to class loader',\n",
       "   '19/06/04 03:52:02 INFO PythonRunner: Times: total = 898, boot = 676, init = 55, finish = 167',\n",
       "   '19/06/04 03:52:02 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 1401 bytes result sent to driver',\n",
       "   '19/06/04 03:52:02 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1515 ms on localhost (executor driver) (1/1)',\n",
       "   '19/06/04 03:52:02 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool ',\n",
       "   '19/06/04 03:52:02 INFO PythonAccumulatorV2: Connected to AccumulatorServer at host: 127.0.0.1 port: 49371',\n",
       "   '19/06/04 03:52:02 INFO DAGScheduler: ResultStage 0 (reduce at <stdin>:8) finished in 1.819 s',\n",
       "   '19/06/04 03:52:02 INFO DAGScheduler: Job 0 finished: reduce at <stdin>:8, took 1.931369 s']}]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "livy.sessions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}