{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-06-13T14:05:55.052373800Z", "start_time": "2024-06-13T14:05:53.599538500Z" } }, "outputs": [], "source": [ "import psycopg2\n", "import pandas as pd\n", "import numpy as np\n", "import warnings\n", "import google.generativeai as genai\n", "from sklearn.base import BaseEstimator\n", "from sklearn.model_selection import train_test_split\n", "warnings.filterwarnings(\"ignore\")\n" ] }, { "cell_type": "code", "execution_count": 3, "outputs": [], "source": [ "conn = psycopg2.connect(dbname='bechdel_test', user='postgres', password='guest')\n", "cur = conn.cursor()\n", "\n", "cur.execute('SELECT * FROM imsdb_scripts JOIN bechdel_ratings ON imsdb_scripts.imdb_id = bechdel_ratings.imdb_id JOIN tmdb_data ON tmdb_data.imdb_id = imsdb_scripts.imdb_id;')\n", "data = pd.DataFrame(cur.fetchall())\n", "df = data.copy()\n", "df.set_index(0, inplace=True)\n", "\n", "cur.execute('SELECT genre.imdb_id, genre FROM genre JOIN imsdb_scripts ON imsdb_scripts.imdb_id = genre.imdb_id;')\n", "genre = pd.DataFrame(cur.fetchall())\n", "cur.close()\n", "conn.close()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:55.878641100Z", "start_time": "2024-06-13T14:05:55.057773700Z" } }, "id": "14e136167bccac0" }, { "cell_type": "code", "execution_count": 4, "outputs": [], "source": [ "for genre_ in genre[1].unique():\n", " df[genre_] = pd.Series()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:55.903387700Z", "start_time": "2024-06-13T14:05:55.880911400Z" } }, "id": "8e360700252225e0" }, { "cell_type": "code", "execution_count": 5, "outputs": [], "source": [ "for row in genre.iterrows():\n", " df[row[1][1]][row[1][0]] = 1" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:56.237431200Z", "start_time": "2024-06-13T14:05:55.898809600Z" } }, "id": "fb80dad8b86a94a6" }, { "cell_type": "code", "execution_count": 6, "outputs": [], "source": [ "df.rename(columns={0:'imdb_id',\n", " 1:'script_date',\n", " 2:'script',\n", " 3:'bechdel_id',\n", " 5:'title',\n", " 6:'release_year',\n", " 7:'bechdel_rating',\n", " 11:'language',\n", " 13:'popularity',\n", " 14:'vote_average',\n", " 15:'vote_count',\n", " 16:'overview'\n", " }, \n", " inplace=True)\n", "df.drop(columns=[4, 8, 9, 10, 12], inplace=True)\n", "df.fillna(0, inplace=True)\n", "df.replace('none', np.nan, inplace=True)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:56.258892400Z", "start_time": "2024-06-13T14:05:56.239236400Z" } }, "id": "c6652299813ffa8c" }, { "cell_type": "code", "execution_count": 7, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(df.drop(columns=['bechdel_rating']), df['bechdel_rating'], test_size=0.234, random_state=42)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:56.287930500Z", "start_time": "2024-06-13T14:05:56.255756800Z" } }, "id": "1e3c4066671fbf5d" }, { "cell_type": "code", "execution_count": 8, "outputs": [], "source": [ "def y_transform(y):\n", " y = pd.DataFrame(y)\n", " y['pass_fail'] = y['bechdel_rating'].map({0:0, 1:0, 2:0, 3:1})\n", " return y\n", "y_train, y_test = y_transform(y_train), y_transform(y_test)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-06-13T14:05:56.290267600Z", "start_time": "2024-06-13T14:05:56.273297300Z" } }, "id": "1ede4a695018215c" }, { "cell_type": "code", "execution_count": 9, "outputs": [ { "data": { "text/plain": " script_date script \\\n0 \n472033 NaN \\n\\n ... \n120780 March 1998 \"Out of Sight\"\\r\\n\\r\\n\\r\\n ... \n1706593 NaN CHRONICLE\\r\\n\\r\\n\\r\\... \n2911666 NaN JOHN WICK\\r\\n\\r\\n\\... \n61722 March 1967 \\t\\t\\t\\t\"THE GRADUATE\"\\r\\n\\r\\n\\r\\n\\t\\t\\t\\tScre... \n... ... ... \n100814 June 1988 \\n\\nS. S. Wilson & Brent Maddock's \"Tremors\"\\n... \n109506 September 1992 The CROW\\r\\n\\r\\n\\tby\\r\\n\\r\\n\\tDavis Schow\\r\\n\\... \n765443 NaN EASTERN PROMISES\\r\\... \n816462 October 2009 NaN \n110148 NaN Interview with the Vampire\\r\\n\\r\\n\\tScreenplay... \n\n bechdel_id title release_year language \\\n0 \n472033 494 9 2009 en \n120780 2247 Out of Sight 1998 en \n1706593 3037 Chronicle 2012 en \n2911666 5897 John Wick 2014 en \n61722 616 Graduate, The 1967 en \n... ... ... ... ... \n100814 1663 Tremors 1990 en \n109506 3820 Crow, The 1994 en \n765443 3069 Eastern Promises 2007 en \n816462 2636 Conan the Barbarian 2011 en \n110148 120 Interview with the Vampire 1994 en \n\n popularity vote_average vote_count \\\n0 \n472033 71.590 6.921 3407 \n120780 24.781 6.682 1203 \n1706593 40.036 6.816 5119 \n2911666 105.961 7.430 18679 \n61722 30.980 7.700 3206 \n... ... ... ... \n100814 77.463 6.896 3105 \n109506 54.672 7.527 3786 \n765443 35.119 7.362 3194 \n816462 35.716 5.299 1792 \n110148 83.427 7.387 5627 \n\n overview ... Thriller \\\n0 ... \n472033 When 9 first comes to life, he finds himself i... ... 1 \n120780 Meet Jack Foley, a smooth criminal who bends t... ... 0 \n1706593 Three high school students make an incredible ... ... 1 \n2911666 Ex-hitman John Wick comes out of retirement to... ... 1 \n61722 Benjamin, a recent college graduate very worri... ... 0 \n... ... ... ... \n100814 Val McKee and Earl Bassett are in a fight for ... ... 0 \n109506 Exactly one year after young rock guitarist Er... ... 1 \n765443 A Russian teenager living in London dies durin... ... 1 \n816462 A quest that begins as a personal vendetta for... ... 0 \n110148 A vampire relates his epic life story of love,... ... 0 \n\n War Comedy Music Western Horror Science Fiction Action \\\n0 \n472033 0 0 0 0 0 1 1 \n120780 0 1 0 0 0 0 0 \n1706593 0 0 0 0 0 1 0 \n2911666 0 0 0 0 0 0 1 \n61722 0 1 0 0 0 0 0 \n... ... ... ... ... ... ... ... \n100814 0 1 0 0 1 1 1 \n109506 0 0 0 0 0 0 1 \n765443 0 0 0 0 0 0 0 \n816462 0 0 0 0 0 0 1 \n110148 0 0 0 0 1 0 0 \n\n Animation History \n0 \n472033 1 0 \n120780 0 0 \n1706593 0 0 \n2911666 0 0 \n61722 0 0 \n... ... ... \n100814 0 0 \n109506 0 0 \n765443 0 0 \n816462 0 0 \n110148 0 0 \n\n[326 rows x 27 columns]", "text/html": "
| \n | script_date | \nscript | \nbechdel_id | \ntitle | \nrelease_year | \nlanguage | \npopularity | \nvote_average | \nvote_count | \noverview | \n... | \nThriller | \nWar | \nComedy | \nMusic | \nWestern | \nHorror | \nScience Fiction | \nAction | \nAnimation | \nHistory | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 472033 | \nNaN | \n\\n\\n ... | \n494 | \n9 | \n2009 | \nen | \n71.590 | \n6.921 | \n3407 | \nWhen 9 first comes to life, he finds himself i... | \n... | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n1 | \n1 | \n0 | \n
| 120780 | \nMarch 1998 | \n\"Out of Sight\"\\r\\n\\r\\n\\r\\n ... | \n2247 | \nOut of Sight | \n1998 | \nen | \n24.781 | \n6.682 | \n1203 | \nMeet Jack Foley, a smooth criminal who bends t... | \n... | \n0 | \n0 | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n
| 1706593 | \nNaN | \nCHRONICLE\\r\\n\\r\\n\\r\\... | \n3037 | \nChronicle | \n2012 | \nen | \n40.036 | \n6.816 | \n5119 | \nThree high school students make an incredible ... | \n... | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n0 | \n
| 2911666 | \nNaN | \nJOHN WICK\\r\\n\\r\\n\\... | \n5897 | \nJohn Wick | \n2014 | \nen | \n105.961 | \n7.430 | \n18679 | \nEx-hitman John Wick comes out of retirement to... | \n... | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n
| 61722 | \nMarch 1967 | \n\\t\\t\\t\\t\"THE GRADUATE\"\\r\\n\\r\\n\\r\\n\\t\\t\\t\\tScre... | \n616 | \nGraduate, The | \n1967 | \nen | \n30.980 | \n7.700 | \n3206 | \nBenjamin, a recent college graduate very worri... | \n... | \n0 | \n0 | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n
| ... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n
| 100814 | \nJune 1988 | \n\\n\\nS. S. Wilson & Brent Maddock's \"Tremors\"\\n... | \n1663 | \nTremors | \n1990 | \nen | \n77.463 | \n6.896 | \n3105 | \nVal McKee and Earl Bassett are in a fight for ... | \n... | \n0 | \n0 | \n1 | \n0 | \n0 | \n1 | \n1 | \n1 | \n0 | \n0 | \n
| 109506 | \nSeptember 1992 | \nThe CROW\\r\\n\\r\\n\\tby\\r\\n\\r\\n\\tDavis Schow\\r\\n\\... | \n3820 | \nCrow, The | \n1994 | \nen | \n54.672 | \n7.527 | \n3786 | \nExactly one year after young rock guitarist Er... | \n... | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n
| 765443 | \nNaN | \nEASTERN PROMISES\\r\\... | \n3069 | \nEastern Promises | \n2007 | \nen | \n35.119 | \n7.362 | \n3194 | \nA Russian teenager living in London dies durin... | \n... | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n
| 816462 | \nOctober 2009 | \nNaN | \n2636 | \nConan the Barbarian | \n2011 | \nen | \n35.716 | \n5.299 | \n1792 | \nA quest that begins as a personal vendetta for... | \n... | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n
| 110148 | \nNaN | \nInterview with the Vampire\\r\\n\\r\\n\\tScreenplay... | \n120 | \nInterview with the Vampire | \n1994 | \nen | \n83.427 | \n7.387 | \n5627 | \nA vampire relates his epic life story of love,... | \n... | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n0 | \n0 | \n
326 rows × 27 columns
\n| \n | script_date | \nscript | \nbechdel_id | \ntitle | \nrelease_year | \nlanguage | \npopularity | \nvote_average | \nvote_count | \noverview | \n... | \nWar | \nComedy | \nMusic | \nWestern | \nHorror | \nScience Fiction | \nAction | \nAnimation | \nHistory | \nembeddings | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 472033 | \nNaN | \n\\n\\n ... | \n494 | \n9 | \n2009 | \nen | \n71.590 | \n6.921 | \n3407 | \nWhen 9 first comes to life, he finds himself i... | \n... | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n1 | \n1 | \n0 | \n[0.009933546, 0.028054273, -0.027433202, 0.011... | \n
| 120780 | \nMarch 1998 | \n\"Out of Sight\"\\r\\n\\r\\n\\r\\n ... | \n2247 | \nOut of Sight | \n1998 | \nen | \n24.781 | \n6.682 | \n1203 | \nMeet Jack Foley, a smooth criminal who bends t... | \n... | \n0 | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n[0.054206412, 0.025492756, 0.036431577, -0.048... | \n
| 1706593 | \nNaN | \nCHRONICLE\\r\\n\\r\\n\\r\\... | \n3037 | \nChronicle | \n2012 | \nen | \n40.036 | \n6.816 | \n5119 | \nThree high school students make an incredible ... | \n... | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n0 | \n[0.0023177029, -0.011539809, -0.0100571215, 0.... | \n
| 2911666 | \nNaN | \nJOHN WICK\\r\\n\\r\\n\\... | \n5897 | \nJohn Wick | \n2014 | \nen | \n105.961 | \n7.430 | \n18679 | \nEx-hitman John Wick comes out of retirement to... | \n... | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n1 | \n0 | \n0 | \n[0.048869684, 0.029862285, -0.0062307036, -0.0... | \n
| 61722 | \nMarch 1967 | \n\\t\\t\\t\\t\"THE GRADUATE\"\\r\\n\\r\\n\\r\\n\\t\\t\\t\\tScre... | \n616 | \nGraduate, The | \n1967 | \nen | \n30.980 | \n7.700 | \n3206 | \nBenjamin, a recent college graduate very worri... | \n... | \n0 | \n1 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n0 | \n[0.04377451, -0.034744043, 0.011599346, -0.005... | \n
5 rows × 28 columns
\nMLPClassifier()In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
MLPClassifier()
| \n | 1 | \n2 | \n3 | \n4 | \n5 | \n6 | \n7 | \n8 | \n9 | \n10 | \n11 | \n12 | \n13 | \n
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n\n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n | \n |
| 9804 | \n14495706 | \nLa Rosace Magique | \n1877 | \n0 | \n766094 | \n14495706 | \nThe Magic Rosette | \nxx | \n1878-05-07 | \n2.194 | \n5.800 | \n19 | \nPraxinoscope strip of a shifting rosette. Seri... | \n
| 9806 | \n12592084 | \nLe singe musicien | \n1878 | \n0 | \n751212 | \n12592084 | \nThe Musician Monkey | \nxx | \n1878-05-07 | \n2.560 | \n5.900 | \n25 | \nA pre-cinematograph colour animation of the mo... | \n
| 9832 | \n8588366 | \nL'homme machine | \n1885 | \n0 | \n585297 | \n8588366 | \nL'Homme Machine | \nxx | \n1885-01-01 | \n1.149 | \n4.629 | \n31 | \nAnimated stick drawings representing a man wal... | \n
| 9614 | \n2075247 | \nMan Walking Around the Corner | \n1887 | \n0 | \n159897 | \n2075247 | \nMan Walking Around a Corner | \nxx | \n1887-08-18 | \n5.529 | \n4.900 | \n80 | \nThe last remaining production of Le Prince's L... | \n
| 9841 | \n7754902 | \nMan Riding Jumping Horse | \n1887 | \n0 | \n1191584 | \n7754902 | \nMan Riding Jumping Horse | \nen | \nNone | \n0.187 | \n4.000 | \n5 | \nA man riding a horse jumps over an obstacle. | \n
| ... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n
| 11302 | \n21235248 | \nGhostbusters: Frozen Empire | \n2024 | \n3 | \n967847 | \n21235248 | \nGhostbusters: Frozen Empire | \nen | \n2024-03-20 | \n603.739 | \n6.671 | \n873 | \nWhen the discovery of an ancient artifact unle... | \n
| 11303 | \n3359350 | \nRoad House | \n2024 | \n3 | \n359410 | \n3359350 | \nRoad House | \nen | \n2024-03-08 | \n483.627 | \n7.024 | \n1810 | \nEx-UFC fighter Dalton takes a job as a bouncer... | \n
| 11317 | \n14539740 | \nGodzilla x Kong: New Empire | \n2024 | \n3 | \n823464 | \n14539740 | \nGodzilla x Kong: The New Empire | \nen | \n2024-03-27 | \n3853.790 | \n7.278 | \n2211 | \nFollowing their explosive showdown, Godzilla a... | \n
| 11318 | \n19356262 | \nDrive-Away Dolls | \n2024 | \n3 | \n957304 | \n19356262 | \nDrive-Away Dolls | \nen | \n2024-02-22 | \n81.501 | \n5.531 | \n208 | \nJamie, an uninhibited free spirit bemoaning ye... | \n
| 11322 | \n26658104 | \nImaginary | \n2024 | \n1 | \n1125311 | \n26658104 | \nImaginary | \nen | \n2024-03-06 | \n154.811 | \n6.210 | \n312 | \nWhen Jessica moves back into her childhood hom... | \n
10133 rows × 13 columns
\n