{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T02:56:32.521682Z",
     "start_time": "2019-03-07T02:56:26.288106Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd, numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T02:56:32.539683Z",
     "start_time": "2019-03-07T02:56:32.526680Z"
    }
   },
   "outputs": [],
   "source": [
    "import lzma,json,gzip\n",
    "import urllib.request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T02:11:53.373625Z",
     "start_time": "2019-03-07T02:11:28.450933Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import load_file, party_normalizer, save_local, load_local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading ep_meps_current.json.xz...\n"
     ]
    }
   ],
   "source": [
    "members=load_file('ep_meps_current.json.xz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T02:22:14.455850Z",
     "start_time": "2019-03-07T02:22:14.426315Z"
    }
   },
   "outputs": [],
   "source": [
    "countries=['Hungary','Romania']\n",
    "eu={}\n",
    "parties={}\n",
    "groups={}\n",
    "names={}\n",
    "for j in members:\n",
    "    z='Constituencies'\n",
    "    w='Groups'    \n",
    "    if z in j:\n",
    "        if j[z][0]['country'] in countries:\n",
    "            if j[z][0]['country'] not in eu:eu[j[z][0]['country']]={}\n",
    "            eu[j[z][0]['country']][j['UserID']]=j\n",
    "            names[j['Name']['full']]=j\n",
    "            for i in j[z]:\n",
    "                if type(i['party'])==str:\n",
    "                    party=i['party']\n",
    "                else:\n",
    "                    party=i['party'][0]\n",
    "                party=str(party)\n",
    "                start=int(str(i['start'])[:4])\n",
    "                end=int(str(i['end'])[:4])\n",
    "                if end==9999:\n",
    "                    end=2019\n",
    "                if party not in parties:\n",
    "                    parties[party]={'min':9999,'max':0}\n",
    "                parties[party]['min']=min(start,parties[party]['min'])\n",
    "                parties[party]['max']=max(end,parties[party]['max'])\n",
    "            if w in j:\n",
    "                for i in j[w]:\n",
    "                    party=i['Organization']\n",
    "                    party=str(party)\n",
    "                    if type(i['groupid'])==str:\n",
    "                        code=i['groupid']\n",
    "                    else:\n",
    "                        code=i['groupid'][0]\n",
    "                    start=int(str(i['start'])[:4])\n",
    "                    end=int(str(i['end'])[:4])\n",
    "                    if end==9999:\n",
    "                        end=2019\n",
    "                    if party not in groups:\n",
    "                        groups[party]={'min':9999,'max':0}\n",
    "                    groups[party]['min']=min(start,groups[party]['min'])\n",
    "                    groups[party]['max']=max(end,groups[party]['max'])\n",
    "                    groups[party]['code']=code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T02:22:17.841191Z",
     "start_time": "2019-03-07T02:22:16.505772Z"
    }
   },
   "outputs": [],
   "source": [
    "save_local(names,'names')\n",
    "save_local(groups,'groups')\n",
    "save_local(parties,'parties')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "del members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading ep_votes.json.xz...\n"
     ]
    }
   ],
   "source": [
    "votes=load_file('ep_votes.json.xz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_allegiance(allegiance,voteid,outcome,name):\n",
    "    if voteid not in allegiance:\n",
    "        allegiance[voteid]={'title':j['title'],'url':j['url'],'ts':j['ts']}\n",
    "    if outcome not in allegiance[voteid]:\n",
    "        allegiance[voteid][outcome]=[]\n",
    "    allegiance[voteid][outcome].append(name)\n",
    "    return allegiance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hungary\n",
      "Romania\n"
     ]
    }
   ],
   "source": [
    "eu_allegiance={}\n",
    "eu_vt={}\n",
    "eu_joint_allegiance={}\n",
    "eu_joint_vt={}\n",
    "for country in countries:\n",
    "    hu=eu[country]\n",
    "    hu_allegiance={}\n",
    "    hu_vt={}\n",
    "    for j in votes:\n",
    "        ts=j['ts']\n",
    "        year=str(ts)[:4]\n",
    "        if year not in hu_vt:hu_vt[year]=[]\n",
    "        if year not in hu_allegiance:hu_allegiance[year]={'name':{},'group':{},'party':{}}  \n",
    "        if year not in eu_joint_vt:eu_joint_vt[year]=[]\n",
    "        if year not in eu_joint_allegiance:eu_joint_allegiance[year]={'name':{},'group':{},'party':{}}    \n",
    "        if j['title'] not in [\"Modification de l'ordre du jour\"]:\n",
    "            for outcome in ['For','Against']:\n",
    "                if outcome in j:\n",
    "                    for group in j[outcome]['groups']:\n",
    "                        for i in group['votes']:\n",
    "                            if i['ep_id'] in hu:\n",
    "                                dummy={}\n",
    "                                dummy['vote']=j['voteid']\n",
    "                                dummy['party']='-'\n",
    "                                for k in hu[i['ep_id']]['Constituencies']:\n",
    "                                    if k['start']<ts<k['end']:\n",
    "                                        dummy['party']=k['party']\n",
    "                                dummy['name']=hu[i['ep_id']]['Name']['full']\n",
    "                                dummy['outcome']=outcome\n",
    "                                dummy['group']=group['group']\n",
    "                                dummy['party']=party_normalizer(dummy['party'])\n",
    "                                dummy['group']=party_normalizer(dummy['group'])\n",
    "                                dummy['title']=j['title']\n",
    "                                dummy['url']=j['url']\n",
    "                                dummy['ts']=ts\n",
    "                                dummy['year']=year\n",
    "                                hu_vt[year].append(dummy)\n",
    "                                eu_joint_vt[year].append(dummy)\n",
    "                                for allegiance_type in ['name','group','party']:\n",
    "                                    hu_allegiance[year][allegiance_type]=\\\n",
    "                                        get_allegiance(hu_allegiance[year][allegiance_type],j['voteid'],\n",
    "                                                   outcome,dummy[allegiance_type])\n",
    "                                    eu_joint_allegiance[year][allegiance_type]=\\\n",
    "                                        get_allegiance(eu_joint_allegiance[year][allegiance_type],j['voteid'],\n",
    "                                                   outcome,dummy[allegiance_type])\n",
    "    eu_allegiance[country]=hu_allegiance\n",
    "    eu_vt[country]=hu_vt\n",
    "    print(country)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "eu_allegiance['Joint']=eu_joint_allegiance\n",
    "eu_vt['Joint']=eu_joint_vt\n",
    "countries=countries+['Joint']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_local(eu_vt,'eu_vt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_local(eu_allegiance,'eu_allegiance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del eu_vt\n",
    "del eu_allegiance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading ep_dossiers.json.xz...\n"
     ]
    }
   ],
   "source": [
    "dossiers=load_file('ep_dossiers.json.xz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading ep_amendments.json.xz...\n"
     ]
    }
   ],
   "source": [
    "#amendments=load_file('ep_amendments.json.xz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "dossiers_by_id={}\n",
    "dossiers_by_report={}\n",
    "for d in dossiers:\n",
    "    dossiers_by_id[d['_id']]=d['procedure']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "voteid_2_dossierid={}\n",
    "for v in votes:\n",
    "    if v['voteid'] not in voteid_2_dossierid:voteid_2_dossierid[v['voteid']]=[]\n",
    "    if 'dossierid' in v:\n",
    "        voteid_2_dossierid[v['voteid']].append(dossiers_by_id[v['dossierid']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_local(voteid_2_dossierid,'voteid_2_dossierid')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}