{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import nltk\n",
"from os import path\n",
"from sklearn.decomposition import PCA\n",
"import sklearn\n",
"from glob import glob\n",
"import codecs\n",
"%matplotlib inline\n",
"plt.style.use('ggplot')\n",
"plt.rcParams['figure.figsize'] = (14,8)\n",
"from sklearn.feature_extraction.text import CountVectorizer"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"class Text(): \n",
" def __init__(self, filename, topWords, chunkSize=5000, maxPieces=1, \n",
" equalPieces=False, maxLen=False): \n",
" with codecs.open(filename, \"r\",encoding='utf-8', errors='ignore') as fdata:\n",
" self.raw = fdata.read().lower()\n",
" self.name = path.splitext(path.basename(filename))[0]\n",
" \n",
" # Use the filename to generate metadata like author name and date of publication. \n",
" # This assumes files named like austen-1811-mansfield.txt\n",
" nameParts = self.name.split('-')\n",
" self.author = nameParts[0]\n",
" self.date = nameParts[1]\n",
" \n",
" self.tokens = nltk.tokenize.word_tokenize(self.raw) \n",
" \n",
" if equalPieces: \n",
" # Break up the text into N equal-sized chunks, where n=maxPieces. \n",
" # Truncate the text at maxLen. \n",
" self.pieces = self.getEqualPieces(self.tokens, maxPieces, maxLen)\n",
" else: \n",
" self.pieces = self.getPieces(self.tokens, chunkSize, maxPieces)\n",
"\n",
" # self.tokens = [word for word in self.tokens if word.isalpha()] # strip punctuation\n",
" self.numPieces = len(self.pieces)\n",
" self.chunkSize = len(self.pieces[0])\n",
" \n",
" if topWords:\n",
" self.freqDists = []\n",
" for piece in self.pieces: \n",
" mc = nltk.FreqDist(piece).most_common(topWords)\n",
" d = {item[0]: item[1] for item in mc}\n",
" self.freqDists.append(d)\n",
" else: \n",
" self.freqDists = [nltk.FreqDist(piece) for piece in self.pieces]\n",
" dfIndex = [self.name + '-' + str(i) for i in range(self.numPieces)]\n",
" self.df = pd.DataFrame(self.freqDists, index=dfIndex).fillna(0)\n",
" \n",
" # I can't figure out how to get sklearn to CountVectorize on already-tokenized text, so \n",
" # putting this on hold for now. \n",
" #vectorizer = CountVectorizer(input='filename')\n",
" #countMatrix = vectorizer.fit_transform(self.pieces).todense()\n",
" #wordLabels = vectorizer.get_feature_names()\n",
" #self.df = pd.DataFrame(countMatrix, columns=matrixLabels, index=textList)\n",
" \n",
" def __repr__(self): \n",
" return \"Text %s: %s pieces of %s words each\" % (self.name, self.numPieces, self.chunkSize)\n",
" \n",
" def __len__(self): \n",
" return len(self.tokens)\n",
" \n",
" def getPieces(self, text, length, num): \n",
" \"\"\" Chunk a text into `num` chunks of size `length`.\"\"\"\n",
" out = [] \n",
" for x in range(num): \n",
" out.append(text[length*x:length*(x+1)])\n",
" return out\n",
" \n",
" def getEqualPieces(self, text, num, maxLen): \n",
" \"\"\" Divide a text into `num` equal pieces, also truncating at maxLen. \"\"\"\n",
" truncated = text[:maxLen]\n",
" length = len(truncated)\n",
" pieceSize = round(length / num)\n",
" return self.getPieces(text, pieceSize, num)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"class TextComparison(): \n",
" def __init__(self, filenames, numPieces=2, topWords=100):\n",
" self.texts = [Text(filename, topWords=topWords) for filename in filenames]\n",
"\n",
" # Determine text with lowest word count. \n",
" # Truncate all other texts to this value. \n",
" cutoff = min(self.wordCounts.values())\n",
" \n",
" # TODO: make this so it doesn't have to instantiate a whole new text object each time. \n",
" # Maybe just get the word count some other way. \n",
" self.texts = [Text(filename, topWords=topWords, maxPieces=numPieces, equalPieces=True, maxLen=cutoff) \n",
" for filename in filenames]\n",
" \n",
" self.dfs = [text.df for text in self.texts]\n",
" self.df = pd.concat(self.dfs).fillna(0)\n",
" self.labels = list(self.df.index)\n",
" self.dates = [int(text.date) for text in self.texts]\n",
" \n",
" @property\n",
" def pieceDates(self):\n",
" dateList = []\n",
" for text in self.texts: \n",
" theseDates = [int(text.date)] * text.numPieces\n",
" dateList += theseDates\n",
" return dateList\n",
"\n",
" @property\n",
" def colors(self, by='date'): \n",
" if by == 'text': \n",
" colors = ['red', 'blue', 'green', 'yellow', 'purple'] * 10\n",
" fullColorVec = []\n",
" for i, text in enumerate(self.texts): \n",
" colorVec = [colors[i]] * text.numPieces\n",
" fullColorVec += colorVec\n",
" if by == 'date': \n",
" fullColorVec = plt.cm.rainbow(self.rescale(self.pieceDates, 0, 1))\n",
" return fullColorVec\n",
" \n",
" def rescale(self, array, low, high):\n",
" minimum = min(array)\n",
" maximum = max(array)\n",
" diff = maximum - minimum\n",
" diffScale = high - low\n",
" return list(map( lambda x: (int(x)-minimum)*(diffScale/diff)+low, array))\n",
" \n",
" @property\n",
" def size(self, by='date'): \n",
" if by == 'date': \n",
" return self.rescale(self.pieceDates, 20, 500)\n",
"\n",
" @property\n",
" def wordCounts(self): \n",
" wordCountsDict = {text.name: len(text) for text in self.texts} \n",
" #pd.Series(wordCountsDict).plot(kind='bar')\n",
" return wordCountsDict\n",
" \n",
" def doPCA(self, labels=False, saveTo=False): \n",
" pca = PCA(n_components=2)\n",
" pcaTable = pca.fit_transform(self.df)\n",
" Xs = [thing[0] for thing in pcaTable]\n",
" Ys = [thing[1] for thing in pcaTable]\n",
" df = pd.DataFrame(pcaTable, index=self.labels)\n",
" df['distance'] = np.sqrt(df[0]**2 + df[1]**2) # Pythagorean theorem \n",
" ax = plt.scatter(df[0], df[1], s=self.size, color=self.colors)\n",
" if labels: \n",
" for i, label in enumerate(self.labels):\n",
" plt.annotate(label, (Xs[i], Ys[i]))\n",
" if saveTo: \n",
" fig = ax.get_figure()\n",
" fig.tight_layout()\n",
" fig.savefig('pca-' + saveTo + '.png') # save it to a file\n",
" return df.sort_values('distance', ascending=False).head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Thomas Mann"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['data/mann-1903-tristan.txt',\n",
" 'data/mann-1903-tonio.txt',\n",
" 'data/mann-1897-kleine-herr.txt',\n",
" 'data/mann-1909-königliche.txt',\n",
" 'data/mann-1943-doktor.txt',\n",
" 'data/mann-1912-tod.txt',\n",
" 'data/mann-1902-gladius.txt',\n",
" 'data/mann-1901-buddenbrooks.txt']"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"textList = glob('data/mann*')\n",
"t = TextComparison(textList)\n",
"textList"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | hemingway-1940-for-whom-the-bell-1 | \n",
" 777.578002 | \n",
" -310.753087 | \n",
" 837.373890 | \n",
"
\n",
" \n",
" | hemingway-1940-for-whom-the-bell-0 | \n",
" 732.158857 | \n",
" -359.839646 | \n",
" 815.807062 | \n",
"
\n",
" \n",
" | hemingway-1952-old-man-and-the-sea-1 | \n",
" 470.000746 | \n",
" 483.108013 | \n",
" 674.013393 | \n",
"
\n",
" \n",
" | hemingway-1952-old-man-and-the-sea-0 | \n",
" 300.983285 | \n",
" 424.574633 | \n",
" 520.436891 | \n",
"
\n",
" \n",
" | hemingway-1926-sun-also-rises-1 | \n",
" -354.193971 | \n",
" -331.715049 | \n",
" 485.271308 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | proust-1925-albertine-0 | \n",
" 1218.883714 | \n",
" -203.294162 | \n",
" 1235.720852 | \n",
"
\n",
" \n",
" | proust-1923-prisonniere-0 | \n",
" 394.899730 | \n",
" 527.083238 | \n",
" 658.606511 | \n",
"
\n",
" \n",
" | proust-1923-prisonniere-1 | \n",
" -146.087649 | \n",
" 564.522948 | \n",
" 583.118993 | \n",
"
\n",
" \n",
" | proust-1913-swann-1 | \n",
" -396.858391 | \n",
" -381.442129 | \n",
" 550.449526 | \n",
"
\n",
" \n",
" | proust-1921-sodome-0 | \n",
" -356.982654 | \n",
" 362.140027 | \n",
" 508.509601 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | bennett-1910-clayhanger-0 | \n",
" 1831.266780 | \n",
" -637.938382 | \n",
" 1939.201743 | \n",
"
\n",
" \n",
" | bennett-1910-clayhanger-1 | \n",
" 1617.798549 | \n",
" -912.341600 | \n",
" 1857.320473 | \n",
"
\n",
" \n",
" | bennett-1902-grand-babylon-0 | \n",
" 277.205253 | \n",
" 1234.851195 | \n",
" 1265.582959 | \n",
"
\n",
" \n",
" | bennett-1902-grand-babylon-1 | \n",
" 332.288596 | \n",
" 1209.293445 | \n",
" 1254.115763 | \n",
"
\n",
" \n",
" | bennett-1902-anna-1 | \n",
" 269.051728 | \n",
" 1183.546827 | \n",
" 1213.742940 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | ward-1881-milly-0 | \n",
" -404.596312 | \n",
" 518.958993 | \n",
" 658.039977 | \n",
"
\n",
" \n",
" | ward-1888-robert-1 | \n",
" 509.411974 | \n",
" 270.602801 | \n",
" 576.824441 | \n",
"
\n",
" \n",
" | ward-1881-milly-1 | \n",
" -231.475069 | \n",
" 492.489367 | \n",
" 544.175049 | \n",
"
\n",
" \n",
" | ward-1900-eleanor-0 | \n",
" 420.663496 | \n",
" -340.238674 | \n",
" 541.036166 | \n",
"
\n",
" \n",
" | ward-1892-grieve-0 | \n",
" 484.829382 | \n",
" 197.966330 | \n",
" 523.689027 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | trollope-1855-warden-0 | \n",
" 1268.582803 | \n",
" -148.330117 | \n",
" 1277.225176 | \n",
"
\n",
" \n",
" | trollope-1857-barchester-0 | \n",
" 938.026670 | \n",
" 538.555508 | \n",
" 1081.635830 | \n",
"
\n",
" \n",
" | trollope-1855-warden-1 | \n",
" 960.834213 | \n",
" -131.649874 | \n",
" 969.811360 | \n",
"
\n",
" \n",
" | trollope-1877-american-senator-1 | \n",
" -877.857701 | \n",
" 299.996680 | \n",
" 927.702620 | \n",
"
\n",
" \n",
" | trollope-1884-old-man-1 | \n",
" -879.666158 | \n",
" -193.646563 | \n",
" 900.728339 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | collins-1875-law-and-the-lady-1 | \n",
" -230.360087 | \n",
" 379.630057 | \n",
" 444.054895 | \n",
"
\n",
" \n",
" | collins-1888-legacy-of-cain-1 | \n",
" -240.087308 | \n",
" 366.886329 | \n",
" 438.460368 | \n",
"
\n",
" \n",
" | collins-1854-hide-and-seek-0 | \n",
" 213.102741 | \n",
" -376.490710 | \n",
" 432.617653 | \n",
"
\n",
" \n",
" | collins-1856-dead-secret-0 | \n",
" 308.770643 | \n",
" -293.333403 | \n",
" 425.891765 | \n",
"
\n",
" \n",
" | collins-1886-guilty-river-1 | \n",
" 207.113563 | \n",
" 356.038516 | \n",
" 411.897381 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | scott-1823-st-ronans-well-1 | \n",
" -353.956658 | \n",
" 1049.564162 | \n",
" 1107.641750 | \n",
"
\n",
" \n",
" | scott-1816-old-mortality-1 | \n",
" 846.746351 | \n",
" 264.552909 | \n",
" 887.111957 | \n",
"
\n",
" \n",
" | scott-1816-antiquary-1 | \n",
" 781.211022 | \n",
" 388.036727 | \n",
" 872.274706 | \n",
"
\n",
" \n",
" | scott-1817-rob-roy-1 | \n",
" 705.388126 | \n",
" 501.925332 | \n",
" 865.737516 | \n",
"
\n",
" \n",
" | scott-1832-count-robert-0 | \n",
" -766.213967 | \n",
" -397.178856 | \n",
" 863.038172 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | dickens-1854-hard-times-1 | \n",
" 1271.034840 | \n",
" 2218.952967 | \n",
" 2557.201954 | \n",
"
\n",
" \n",
" | dickens-1854-hard-times-0 | \n",
" 1092.526295 | \n",
" 2249.074311 | \n",
" 2500.389762 | \n",
"
\n",
" \n",
" | dickens-1861-great-expectations-1 | \n",
" 1799.528701 | \n",
" -658.096758 | \n",
" 1916.088435 | \n",
"
\n",
" \n",
" | dickens-1861-great-expectations-0 | \n",
" 1642.802360 | \n",
" -684.903022 | \n",
" 1779.857226 | \n",
"
\n",
" \n",
" | dickens-1853-bleak-house-1 | \n",
" 1553.506626 | \n",
" -776.504789 | \n",
" 1736.762080 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | cather-1927-death-comes-for-the-archbishop-0 | \n",
" 231.510416 | \n",
" -95.846651 | \n",
" 250.566664 | \n",
"
\n",
" \n",
" | cather-1912-alexanders-bridge-1 | \n",
" -205.490685 | \n",
" -128.083706 | \n",
" 242.140161 | \n",
"
\n",
" \n",
" | cather-1931-shadows-on-the-rock-0 | \n",
" 230.910182 | \n",
" 19.769101 | \n",
" 231.754891 | \n",
"
\n",
" \n",
" | cather-1926-my-mortal-enemy-1 | \n",
" -228.179881 | \n",
" -11.996383 | \n",
" 228.495014 | \n",
"
\n",
" \n",
" | cather-1940-sapphira-and-the-slave-girl-0 | \n",
" -50.311710 | \n",
" 181.835163 | \n",
" 188.667153 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | meredith-1856-the-shaving-of-shagpat-1 | \n",
" 469.875705 | \n",
" 224.152468 | \n",
" 520.603022 | \n",
"
\n",
" \n",
" | meredith-1856-the-shaving-of-shagpat-0 | \n",
" 404.175379 | \n",
" 230.485803 | \n",
" 465.275662 | \n",
"
\n",
" \n",
" | meredith-1877-the-house-on-the-beach-1 | \n",
" -310.171929 | \n",
" 42.648531 | \n",
" 313.090279 | \n",
"
\n",
" \n",
" | meredith-1865-rhoda-fleming-1 | \n",
" -269.253450 | \n",
" 157.141401 | \n",
" 311.754455 | \n",
"
\n",
" \n",
" | meredith-1859-the-ordeal-of-richard-feverel-1 | \n",
" -291.536054 | \n",
" 61.514307 | \n",
" 297.955165 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | conrad-1904-nostromo-0 | \n",
" 592.514997 | \n",
" 253.415703 | \n",
" 644.432728 | \n",
"
\n",
" \n",
" | conrad-1919-the-arrow-of-gold-1 | \n",
" -558.160780 | \n",
" 53.100014 | \n",
" 560.680897 | \n",
"
\n",
" \n",
" | conrad-1917-the-shadow-line-0 | \n",
" -518.430499 | \n",
" 131.287116 | \n",
" 534.795745 | \n",
"
\n",
" \n",
" | conrad-1895-almayers-folly-0 | \n",
" 524.877182 | \n",
" -55.002460 | \n",
" 527.751198 | \n",
"
\n",
" \n",
" | conrad-1904-nostromo-1 | \n",
" 460.254331 | \n",
" 242.720993 | \n",
" 520.334056 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | genet-1986-captif-amoreux-1 | \n",
" 1974.868125 | \n",
" -11.667728 | \n",
" 1974.902592 | \n",
"
\n",
" \n",
" | genet-1986-captif-amoreux-0 | \n",
" 1884.846172 | \n",
" -80.660954 | \n",
" 1886.571303 | \n",
"
\n",
" \n",
" | genet-1942-notre-dame-0 | \n",
" -203.910840 | \n",
" 846.561123 | \n",
" 870.772855 | \n",
"
\n",
" \n",
" | genet-1942-notre-dame-1 | \n",
" -278.196147 | \n",
" 774.088522 | \n",
" 822.560718 | \n",
"
\n",
" \n",
" | genet-1949-journal-du-voleur-1 | \n",
" -574.217836 | \n",
" -531.655056 | \n",
" 782.549182 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | james-1899-awkward-age-1 | \n",
" 1365.867641 | \n",
" -413.504271 | \n",
" 1427.088013 | \n",
"
\n",
" \n",
" | james-1911-outcry-0 | \n",
" 1419.785990 | \n",
" -80.771879 | \n",
" 1422.081698 | \n",
"
\n",
" \n",
" | james-1911-outcry-1 | \n",
" 1370.935925 | \n",
" 5.254123 | \n",
" 1370.945993 | \n",
"
\n",
" \n",
" | james-1888-reverberator-1 | \n",
" 1143.051153 | \n",
" -613.465954 | \n",
" 1297.268829 | \n",
"
\n",
" \n",
" | james-1899-awkward-age-0 | \n",
" 1219.482492 | \n",
" -404.127565 | \n",
" 1284.700992 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" 0 | \n",
" 1 | \n",
" distance | \n",
"
\n",
" \n",
" \n",
" \n",
" | hemingway-1940-for-whom-the-bell-1 | \n",
" 777.578002 | \n",
" -310.753087 | \n",
" 837.373890 | \n",
"
\n",
" \n",
" | hemingway-1940-for-whom-the-bell-0 | \n",
" 732.158857 | \n",
" -359.839646 | \n",
" 815.807062 | \n",
"
\n",
" \n",
" | hemingway-1952-old-man-and-the-sea-1 | \n",
" 470.000746 | \n",
" 483.108013 | \n",
" 674.013393 | \n",
"
\n",
" \n",
" | hemingway-1952-old-man-and-the-sea-0 | \n",
" 300.983285 | \n",
" 424.574633 | \n",
" 520.436891 | \n",
"
\n",
" \n",
" | hemingway-1926-sun-also-rises-1 | \n",
" -354.193971 | \n",
" -331.715049 | \n",
" 485.271308 | \n",
"
\n",
" \n",
"
\n",
"