{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br></br>\n",
    "# **gensim | word2vec**\n",
    "<br></br>\n",
    "## **1 데이터 전처리**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from txtutil import txtnoun\n",
    "# 2015 ~ 2018 지속가능 경영 보고서 Token을 수집\n",
    "filelist = glob.glob('./data/kr-Report_201?.txt')\n",
    "print(filelist)\n",
    "\n",
    "# 불러온 Document 명사Token만 추출\n",
    "skiplist = {'갤러시':'갤럭시', '가치창출':'가치창출'}\n",
    "texts    = [txtnoun(file, skip=skiplist)  for file in filelist]\n",
    "texts    = \" \".join(texts)\n",
    "texts[:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 명사 Token 작업된 자료를 ssResport.txt 로 저장 \n",
    "texts_file = './data/ssResport.txt'\n",
    "with open(texts_file, 'w', encoding='utf-8') as file:\n",
    "    file.write(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! cat ./data/ssResport.txt | head -n 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br></br>\n",
    "## **2 Word 2 vec 객체 만들기**\n",
    "gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! pip3 install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "texts_file = './data/ssResport.txt'\n",
    "\n",
    "from gensim.models import word2vec\n",
    "data  = word2vec.LineSentence(texts_file)\n",
    "model = word2vec.Word2Vec(data, size=200, window=2, min_count=20, hs=1,\n",
    "                          workers=4, iter=100, sg=1)\n",
    "model.save(\"./data/ssReport.model\")\n",
    "print(\"model saved.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br></br>\n",
    "## **3 저장된 객체 활용**\n",
    "gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%who"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import word2vec\n",
    "model = word2vec.Word2Vec.load('./data/ssReport.model')\n",
    "len(model.wv.vocab.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(model.wv.index2word)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar(positive=['삼성전자'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar(negative=['삼성전자'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar(positive=['글로벌'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar(negative=['글로벌'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.most_similar(positive=['삼성전자','경영활동'], \n",
    "                      negative=['근무환경']) # 담당자, 직원"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br></br>\n",
    "## **04 Visulaization**\n",
    "gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(model.wv.vocab.keys())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.wv.vocab : { word: object of numeric vector }\n",
    "vocab  = list(model.wv.vocab)\n",
    "X      = model[vocab]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from sklearn.manifold import TSNE\n",
    "tsne   = TSNE(n_components = \"=Quiz!=\")\n",
    "X_tsne = tsne.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(X_tsne, \n",
    "                  index = vocab, \n",
    "                  columns=['x', 'y'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import rc\n",
    "rc('font', family=['NanumGothic','Malgun Gothic'])\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "fig = plt.figure(figsize=(12,12))\n",
    "ax  = fig.add_subplot(1, 1, 1)\n",
    "ax.scatter(df['x'], df['y'])\n",
    "for word, pos in df.iterrows():\n",
    "    ax.annotate(word, pos)\n",
    "plt.grid(True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}