{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Cosine Similarity (Version B)\n",
"\n",
"- Loads embeddings (pre-computed from texts)\n",
"- Computes **mean of embedding**-arrays\n",
"- Compares two similarity approaches:\n",
" - Computes **cosine-similarity** of two embeddings-vectors\n",
" - Computes **difference value** of two embeddings-vectors\n",
"- Results/observations:\n",
" - **Both approaches** compute **similar results** (cell 6)\n",
" - Even on **small similarity** (large distance / large difference) between mean-embeddings and text-embeddings, the underlying text versions can be **very similar**.\n",
" - Small similarity (large difference) in Wikipedia texts is often **based on additional contents**, not modifications -> Consider for **drift detection**"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# Configuration\n",
"\n",
"# https://hobbitdata.informatik.uni-leipzig.de/EML4U/2021-02-10-Wikipedia-Texts/\n",
"source_texts_directory = \"/home/eml4u/EML4U/data/corpus/2021-02-10-wikipedia-texts/\"\n",
"# https://hobbitdata.informatik.uni-leipzig.de/EML4U/2021-04-07-Wikipedia-Embeddings/\n",
"embeddings_directory = \"/home/eml4u/EML4U/data/wikipedia-embeddings/\"\n",
"\n",
"# points of time\n",
"id_a = \"20100408\"\n",
"id_b = \"20201101\"\n",
"# category ids\n",
"id_american = \"american-films\"\n",
"id_british = \"british-films\"\n",
"id_indian = \"indian-films\"\n",
"# file ids\n",
"id_american_a = id_a + \"-\" + id_american\n",
"id_american_b = id_b + \"-\" + id_american\n",
"id_british_a = id_a + \"-\" + id_british\n",
"id_british_b = id_b + \"-\" + id_british\n",
"id_indian_a = id_a + \"-\" + id_indian\n",
"id_indian_b = id_b + \"-\" + id_indian"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"numpy: 1.19.2\n",
"sklearn: 0.23.2\n"
]
}
],
"source": [
"# Imports\n",
"\n",
"import numpy\n",
"print(\"numpy: \" + numpy.version.version)\n",
"\n",
"import sklearn\n",
"import sklearn.metrics\n",
"print(\"sklearn: \" + sklearn.__version__)\n",
"\n",
"# Class instance to access data (wp texts, pre-computed embeddings)\n",
"import data_access\n",
"data_accessor = data_access.DataAccess(source_texts_directory, embeddings_directory)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/home/eml4u/EML4U/data/wikipedia-embeddings/20100408-british-films.txt\n",
"(2147, 768) <class 'numpy.ndarray'>\n",
"/home/eml4u/EML4U/data/wikipedia-embeddings/20201101-british-films.txt\n",
"(2147, 768) <class 'numpy.ndarray'>\n",
"\n",
"<class 'numpy.ndarray'> (768,) BritishA\n",
"<class 'numpy.ndarray'> (768,) BritishB\n"
]
}
],
"source": [
"# Load embeddings\n",
"embeddings_british_a = data_accessor.load_embeddings(id_british_a)\n",
"embeddings_british_b = data_accessor.load_embeddings(id_british_b)\n",
"print()\n",
"\n",
"# Compute means\n",
"def get_mean(embeddings, note = \"\", printinfo = True):\n",
" mean = numpy.mean(embeddings, axis=0)\n",
" if printinfo:\n",
" print(str(type(mean)) + \" \" + str(mean.shape) + \" \" + note)\n",
" return mean\n",
"\n",
"mean_british_a = get_mean(embeddings_british_a, \"BritishA\")\n",
"mean_british_b = get_mean(embeddings_british_b, \"BritishB\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# Cosine similarity\n",
"def get_pairwise_cosine_similarity(a, b, note = \"\", printinfo = True):\n",
" if printinfo:\n",
" print(str(type(a)) + \" \" + str(a.shape) + \"\\n\" + str(type(b)) + \" \" + str(b.shape))\n",
" cosSim = sklearn.metrics.pairwise.cosine_similarity(a, b, dense_output=True)[0][0]\n",
" if printinfo:\n",
" print(str(cosSim) + \" \" + note)\n",
" return cosSim\n",
"\n",
"similarities_a = []\n",
"for i in range(len(mean_british_a)):\n",
" similarities_a.append((i, get_pairwise_cosine_similarity( mean_british_a.reshape(1, -1), embeddings_british_a[i].reshape(1, -1), \"\", False )))\n",
"smallest_similarities_a = sorted(similarities_a, key=lambda tup: tup[1], reverse=False)\n",
"\n",
"similarities_b = []\n",
"for i in range(len(mean_british_b)):\n",
" similarities_b.append((i, get_pairwise_cosine_similarity( mean_british_b.reshape(1, -1), embeddings_british_b[i].reshape(1, -1), \"\", False )))\n",
"smallest_similarities_b = sorted(similarities_b, key=lambda tup: tup[1], reverse=False)\n",
"\n",
"similarities_direct = []\n",
"for i in range(len(embeddings_british_a)):\n",
" similarities_direct.append((i, get_pairwise_cosine_similarity( embeddings_british_a[i].reshape(1, -1), embeddings_british_b[i].reshape(1, -1), \"\", False )))\n",
"smallest_similarities_direct = sorted(similarities_direct, key=lambda tup: tup[1], reverse=False)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.0\n"
]
}
],
"source": [
"# Differences of arrays as one value\n",
"def differenceValue(a, b):\n",
" x = 0\n",
" for i in range(len(a)):\n",
" x += abs(a[i] - b[i])\n",
" return x;\n",
"# Test .1 + .2 + .3 + .4 = 1\n",
"if True:\n",
" print(differenceValue(numpy.array([1,2,3,0]), numpy.array([1.1,2.2,3.3,-0.4])))\n",
"\n",
"\n",
"# Compute difference values between embeddings of single texts and mean-embeddings\n",
"# Array:\n",
"# [0] index (to look up source texts)\n",
"# [1] difference to mean t1 (== A)\n",
"# [2] difference to mean t2 (== B)\n",
"differences = []\n",
"for i in range(len(mean_british_a)):\n",
" differences.append((i, differenceValue(mean_british_a, embeddings_british_a[i]), differenceValue(mean_british_b, embeddings_british_b[i])))\n",
"\n",
"# Sort by largest difference\n",
"largest_differences_a = sorted(differences, key=lambda tup: tup[1], reverse=True)\n",
"largest_differences_b = sorted(differences, key=lambda tup: tup[2], reverse=True)\n",
"\n",
"\n",
"# Explore embeddings of two points of time directly\n",
"differences_direct = []\n",
"for i in range(len(mean_british_a)):\n",
" differences_direct.append((i, differenceValue(embeddings_british_a[i], embeddings_british_b[i])))\n",
"\n",
"# Sort\n",
"largest_differences_direct = sorted(differences_direct, key=lambda tup: tup[1], reverse=True)\n",
"\n",
"\n",
"# Print source texts\n",
"def print_source_text(directory, category_id, index):\n",
" print()\n",
" print(\"Category: \" + category_id)\n",
" print(\"Index: \" + str(index))\n",
" file = data_accessor.get_embeddings_dict_filename(category_id, index);\n",
" print(\"File: \")\n",
" print(data_accessor.read_source_text(directory, file))\n",
" print()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Smallest cosine similarity to mean of A\n",
"(721, 0.8095643688026439)\n",
"(333, 0.8195543132287988)\n",
"(680, 0.820176887350935)\n",
"...\n",
"(391, 0.9703187666616886)\n",
"(393, 0.9757224095034057)\n",
"\n",
"Largest difference values to mean of A\n",
"(721, 127.63656949018497, 63.07215986661895)\n",
"(333, 126.23330120916125, 126.63447396310393)\n",
"(680, 122.11133584967376, 129.0560627009414)\n",
"...\n",
"(391, 40.37050334666524, 45.36463767773769)\n",
"(393, 39.05389511333612, 56.23254257812948)\n",
"\n",
"\n",
"Smallest cosine similarity to mean of B\n",
"(680, 0.7901596223863818)\n",
"(333, 0.8048592466102065)\n",
"(179, 0.806587230956004)\n",
"...\n",
"(334, 0.973834448611832)\n",
"(309, 0.9741528939831201)\n",
"\n",
"Largest difference values to mean of B\n",
"(179, 114.6978323360855, 131.07256570494872)\n",
"(680, 122.11133584967376, 129.0560627009414)\n",
"(381, 88.82879530042067, 127.66585960996093)\n",
"...\n",
"(334, 51.76971304423001, 39.533962588139545)\n",
"(309, 56.43851055612732, 39.04979437720267)\n",
"\n",
"\n",
"Smallest cosine similarity (direct)\n",
"(1047, 0.7142031344871039)\n",
"(864, 0.7232851659170279)\n",
"(1442, 0.7256858182916857)\n",
"...\n",
"(1604, 0.999699356760097)\n",
"(1635, 0.9997250982866731)\n",
"\n",
"Largest difference values (direct)\n",
"(721, 152.96771019252628)\n",
"(610, 141.938466045307)\n",
"(409, 141.68865489200107)\n",
"...\n",
"(422, 5.0086785865423735)\n",
"(435, 4.515663030353608)\n",
"\n"
]
}
],
"source": [
"print(\"Smallest cosine similarity to mean of A\")\n",
"print(smallest_similarities_a[0])\n",
"print(smallest_similarities_a[1])\n",
"print(smallest_similarities_a[2])\n",
"print(\"...\")\n",
"print(smallest_similarities_a[len(smallest_similarities_a)-2])\n",
"print(smallest_similarities_a[len(smallest_similarities_a)-1])\n",
"print()\n",
"\n",
"print(\"Largest difference values to mean of A\")\n",
"print(largest_differences_a[0])\n",
"print(largest_differences_a[1])\n",
"print(largest_differences_a[2])\n",
"print(\"...\")\n",
"print(largest_differences_a[len(largest_differences_a)-2])\n",
"print(largest_differences_a[len(largest_differences_a)-1])\n",
"print(\"\\n\")\n",
"\n",
"print(\"Smallest cosine similarity to mean of B\")\n",
"print(smallest_similarities_b[0])\n",
"print(smallest_similarities_b[1])\n",
"print(smallest_similarities_b[2])\n",
"print(\"...\")\n",
"print(smallest_similarities_b[len(smallest_similarities_b)-2])\n",
"print(smallest_similarities_b[len(smallest_similarities_b)-1])\n",
"print()\n",
"\n",
"print(\"Largest difference values to mean of B\")\n",
"print(largest_differences_b[0])\n",
"print(largest_differences_b[1])\n",
"print(largest_differences_b[2])\n",
"print(\"...\")\n",
"print(largest_differences_b[len(largest_differences_b)-2])\n",
"print(largest_differences_b[len(largest_differences_b)-1])\n",
"print(\"\\n\")\n",
"\n",
"print(\"Smallest cosine similarity (direct)\")\n",
"print(smallest_similarities_direct[0])\n",
"print(smallest_similarities_direct[1])\n",
"print(smallest_similarities_direct[2])\n",
"print(\"...\")\n",
"print(smallest_similarities_direct[len(smallest_similarities_direct)-2])\n",
"print(smallest_similarities_direct[len(smallest_similarities_direct)-1])\n",
"print()\n",
"\n",
"print(\"Largest difference values (direct)\")\n",
"print(largest_differences_direct[0])\n",
"print(largest_differences_direct[1])\n",
"print(largest_differences_direct[2])\n",
"print(\"...\")\n",
"print(largest_differences_direct[len(largest_differences_direct)-2])\n",
"print(largest_differences_direct[len(largest_differences_direct)-1])\n",
"print()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"333: Very similar, even if difference to mean of A is large\n",
"\n",
"Category: british-films\n",
"Index: 333\n",
"File: \n",
"/home/eml4u/EML4U/data/corpus/2021-02-10-wikipedia-texts/20100408-british-films/The_Tide_of_Traffic.txt\n",
"The Tide of Traffic is a 1972 short documentary film directed by Derek\n",
"Williams. It was nominated for an Academy Award for Best Documentary\n",
"Short. \n",
"References\n",
"External links\n",
"- \n",
"Category:1972 films Category:British films Category:English-language\n",
"films Category:British documentary films Category:Short films\n",
"\n",
"\n",
"\n",
"Category: british-films\n",
"Index: 333\n",
"File: \n",
"/home/eml4u/EML4U/data/corpus/2021-02-10-wikipedia-texts/20201101-british-films/The_Tide_of_Traffic.txt\n",
"The Tide of Traffic is a 1972 British short documentary film directed by\n",
"Derek Williams. It was nominated for an Academy Award for Best\n",
"Documentary Short.\n",
"References\n",
"External links\n",
"- Watch The Tide of Traffic at BP Video Library\n",
"- \n",
"Category:1972 films Category:1972 documentary films Category:1972 short\n",
"films Category:British films Category:English-language films\n",
"Category:British documentary films Category:Short documentary films\n",
"\n",
"\n"
]
}
],
"source": [
"# Explore underlying texts (Largest difference values to mean of A)\n",
"if False:\n",
" print(\"721: As expected\")\n",
" print_source_text(id_british_a, id_british, largest_differences_a[0][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_a[0][0])\n",
"if True:\n",
" print(\"333: Very similar, even if difference to mean of A is large\")\n",
" print_source_text(id_british_a, id_british, largest_differences_a[1][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_a[1][0])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"179: Very similar, even if difference to mean of B is large\n",
"\n",
"Category: british-films\n",
"Index: 179\n",
"File: \n",
"/home/eml4u/EML4U/data/corpus/2021-02-10-wikipedia-texts/20100408-british-films/The_World_Is_Rich.txt\n",
"The World Is Rich is a 1947 documentary film directed by Paul Rotha. It\n",
"was nominated for an Academy Award for Best Documentary Feature.\n",
"References\n",
"External links\n",
"- \n",
"fr:The World Is Rich\n",
"Category:1947 films Category:British films Category:English-language\n",
"films Category:British documentary films Category:Black-and-white films\n",
"Category:Films directed by Paul Rotha\n",
"\n",
"\n",
"\n",
"Category: british-films\n",
"Index: 179\n",
"File: \n",
"/home/eml4u/EML4U/data/corpus/2021-02-10-wikipedia-texts/20201101-british-films/The_World_Is_Rich.txt\n",
"The World Is Rich is a 1947 British documentary film directed by Paul\n",
"Rotha. It was nominated for an Academy Award for Best Documentary\n",
"Feature..\n",
"References\n",
"External links\n",
"- \n",
"Category:1947 films Category:1947 documentary films Category:British\n",
"films Category:English-language films Category:British documentary films\n",
"Category:Black-and-white documentary films Category:Films directed by\n",
"Paul Rotha Category:British black-and-white films\n",
"\n",
"\n"
]
}
],
"source": [
"# Explore underlying texts (Largest difference values to mean of B)\n",
"if True:\n",
" print(\"179: Very similar, even if difference to mean of B is large\")\n",
" print_source_text(id_british_a, id_british, largest_differences_b[0][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_b[0][0])\n",
"if False:\n",
" print(\"680: Very similar, even if difference to mean of B is large\")\n",
" print_source_text(id_british_a, id_british, largest_differences_b[1][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_b[1][0])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# Explore underlying texts (direct difference values)\n",
"if False:\n",
" print(\"721: As expected, different\")\n",
" print_source_text(id_british_a, id_british, largest_differences_direct[0][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_direct[0][0])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"# Explore underlying texts (direct difference values)\n",
"if False:\n",
" print(\"435: As expected, similar\")\n",
" print_source_text(id_british_a, id_british, largest_differences_direct[len(largest_differences_direct)-1][0])\n",
" print_source_text(id_british_b, id_british, largest_differences_direct[len(largest_differences_direct)-1][0])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# Explore underlying texts (direct similarity values)\n",
"if False:\n",
" print(\"1047: As expected, different\")\n",
" print_source_text(id_british_a, id_british, smallest_similarities_direct[0][0])\n",
" print_source_text(id_british_b, id_british, smallest_similarities_direct[0][0])\n",
"if False:\n",
" print(\"864: As expected, different\")\n",
" print_source_text(id_british_a, id_british, smallest_similarities_direct[1][0])\n",
" print_source_text(id_british_b, id_british, smallest_similarities_direct[1][0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python (EML4U)",
"language": "python",
"name": "eml4u"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}