{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "

\n", "# **Project4 | word2vec**\n", "

\n", "## **1 데이터 전처리**" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# 살인의 추억 텍스트 불러오기\n", "filename = './data/movie_memories_of_murder_2003.txt'\n", "with open(filename, 'r') as f:\n", " texts = f.read()\n", "texts[:500]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from txtutil import txtnoun\n", "skips = {'두만':'박두만', '태윤':'서태윤', '용구':'조용구', '귀옥':'권귀옥', \n", " '희봉':'구희봉', '동철':'신동철', '광호':'백광호', '병순':'조병순', \n", " '해일':'박해일', '광호의':'백광호', '백광호의':'백광호'}\n", "%time texts = txtnoun(filename, skip=skips, tags=['Noun'])\n", "texts[:500]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# 명사 Token 작업된 자료를 ssResport.txt 로 저장 \n", "texts_file = './data/mom_noun_script.txt'\n", "with open(texts_file, 'w', encoding='utf-8') as file:\n", " file.write(texts)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ! cat ./data/ssResport.txt | head -n 10" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "

\n", "## **2 Word 2 vec 객체 만들기**\n", "gensim" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ! pip3 install gensim" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "texts_file = './data/mom_noun_script.txt'\n", "\n", "from gensim.models import word2vec\n", "data = word2vec.LineSentence(texts_file)\n", "model = word2vec.Word2Vec(data, size=30, window=2, min_count=10, hs=1,\n", " workers=4, iter=100, sg=1)\n", "model.save(\"./data/mom_script.model\")\n", "print(\"model saved.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "

\n", "## **3 저장된 모델 불러오기 및 확인**\n", "gensim" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%reset" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%who" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from gensim.models import word2vec\n", "model = word2vec.Word2Vec.load('./data/mom_script.model')\n", "len(model.wv.vocab.keys())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "list(model.wv.index2word)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "

\n", "## **4 Word2Vec 모델 내용 확인**\n", "gensim" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['범인'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(negative=['범인'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['피해자'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(negative=['피해자'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['박두만'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [], "source": [ "model.wv.most_similar(positive=['서태윤'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": false }, "outputs": [], "source": [ "model.wv.most_similar(positive=['조용구'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['박두만','서태윤'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['박두만','서태윤'],\n", " negative=['피해자'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['박두만','서태윤'],\n", " negative=['박해일'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model.wv.most_similar(positive=['박두만','서태윤'],\n", " negative=['범인'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "

\n", "## **5 Visulaization**\n", "gensim" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "list(model.wv.vocab.keys())[:10]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# model.wv.vocab : { word: object of numeric vector }\n", "vocab = list(model.wv.vocab)\n", "X = model[vocab]\n", "X.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%time\n", "from sklearn.manifold import TSNE\n", "tsne = TSNE(n_components= 2)\n", "X_tsne = tsne.fit_transform(X)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "df = pd.DataFrame(X_tsne, \n", " index = vocab, \n", " columns=['x', 'y'])\n", "df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "from matplotlib import rc\n", "rc('font', family=['NanumGothic','Malgun Gothic'])\n", "\n", "import matplotlib.pyplot as plt\n", "fig = plt.figure(figsize=(12,12))\n", "ax = fig.add_subplot(1, 1, 1)\n", "ax.scatter(df['x'], df['y'])\n", "for word, pos in df.iterrows():\n", " ax.annotate(word, pos, fontsize=15)\n", "plt.grid(True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.3" } }, "nbformat": 4, "nbformat_minor": 2 }