{ "metadata": { "signature": "sha256:028dc5e5e0ad07086953ec28d7dde90d1cbc4af91026f4da53393013fef815b8" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Topic Modeling on Turkish News\n", "by [Talha Oz](http://www.mli.gmu.edu/toz/wordpress/)\n", "\n", "Details on the project is available on [GitHub](https://github.com/oztalha/News-Commentary-Tweets-of-Elites)" ] }, { "cell_type": "code", "collapsed": false, "input": [ "cd 'C:\\Users\\Talha\\Documents\\WinPython\\projelerim\\news\\News-Commentary-Tweets-of-Elites'" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "C:\\Users\\Talha\\Documents\\WinPython\\projelerim\\news\\News-Commentary-Tweets-of-Elites\n" ] } ], "prompt_number": 1 }, { "cell_type": "code", "collapsed": false, "input": [ "import pandas as pd\n", "from gensim import corpora, models\n", "from nltk.corpus import stopwords\n", "import nltk\n", "import snowballstemmer\n", "import re\n", "from operator import itemgetter" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 2 }, { "cell_type": "code", "collapsed": false, "input": [ "df = pd.read_csv('data/TR-news.csv',usecols=['dt','title','newstxt'],parse_dates=[0])\n", "df = df[df['newstxt'].str.contains(\"Amerika|ABD|Obama\")]\n", "documents = df.newstxt.tolist()\n", "df.sort(['dt']).to_csv('data/amerika-news.csv',encoding='utf-8',index=False,columns=['dt','title'])\n", "df.sort(['dt'])" ], "language": "python", "metadata": {}, "outputs": [ { "html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
dttitlenewstxt
19162013-01-14 1979 \u0130ran\u2019\u0131na \u00f6d\u00fcl ya\u011fd\u0131 \u2013 nediyor.com \\nOscar\u2019dan sonra sinema d\u00fcnyas\u0131n\u0131n en prestij...
19172013-01-14 Sa\u011fl\u0131k Bakanl\u0131\u011f\u0131\u2019ndan Refiye a\u00e7\u0131klamas\u0131 \u2013 nedi... \\nMilyonda bir g\u00f6r\u00fclen \u201cGardner sendromu\u201d hast...
18522013-01-16 \u00dcnl\u00fc ressam Do\u011fan\u00e7ay hayat\u0131n\u0131 kaybetti \u2013 nediy... \\n\u00dcnl\u00fc ressam Burhan Do\u011fan\u00e7ay bu sabah tedavi ...
18512013-01-16 Mehmet Ali Birand hastaneye kald\u0131r\u0131ld\u0131 \u2013 nediy... \\nKanal D Haber Grup Ba\u015fkan\u0131 ve anchorman\u2019\u0131 de...
18122013-01-17 Gazetecili\u011fin ikonu Birand\u2019\u0131 kaybettik \u2013 nediy... \\nKanal D Haber Grup Ba\u015fkan\u0131 Mehmet Ali Birand...
18252013-01-17 \u00d6l\u00fcm haberine Umur Birand\u2019tan yalanlama \u2013 nedi... \\nKanal D Haber Grup Ba\u015fkan\u0131 Mehmet Ali Birand...
17952013-01-18 Birand\u2019\u0131n \u00f6l\u00fcm\u00fcnde ihmal \u015f\u00fcphesi \u2013 nediyor.com \\nDuayen gazeteci Mehmet Ali Birand\u2019\u0131n Sal\u0131 g\u00fc...
17962013-01-18 Armstrong sonunda dopingi itiraf etti \u2013 nediyo... \\nAmerika\u2019n\u0131n g\u00fcndemi Lance Armstrong. \u00dcnl\u00fc bi...
17882013-01-19 Birand son yolculu\u011funa u\u011furland\u0131 \u2013 nediyor.com \\nPer\u015fembe g\u00fcn\u00fc vefat eden duayen gazeteci Meh...
17612013-01-21 Patriotlar T\u00fcrkiye\u2019de \u2013 nediyor.com \\nT\u00fcrkiye\u2019nin Suriye s\u0131n\u0131r\u0131na konu\u015fland\u0131rmak \u00fc...
70 2013-01-22 Hatay\u2019da NATO askerine \u00e7uval ge\u00e7irme giri\u015fimi ... \\nHatay\u2019\u0131n \u0130skenderun il\u00e7esinde T\u00fcrkiye Gen\u00e7li...
17442013-01-22 Obama ikinci kez yemin etti \u2013 nediyor.com \\nABD Ba\u015fkan\u0131 Barack Obama, ikinci ba\u015fkanl\u0131k d...
16472013-01-25 Samatya\u2019daki sald\u0131r\u0131lar\u0131n faili ayn\u0131 ki\u015fi \u2013 ne... \\n\u0130stanbul\u2019da ge\u00e7ti\u011fimiz Aral\u0131k ay\u0131ndan bu yan...
15512013-01-29 \u201c\u015eangay Be\u015flisi\u201d a\u00e7\u0131l\u0131m\u0131na ABD\u2019den g\u00fcl\u00fcmseme \u2013... \\nBa\u015fbakan Erdo\u011fan\u2019\u0131n AB\u2019ye alternatif olarak ...
56 2013-01-30 \u0130srail u\u00e7aklar\u0131 Suriye\u2019yi vurdu \u2013 nediyor.com \\nSuriye, \u0130srail\u2019e ait sava\u015f u\u00e7aklar\u0131n\u0131n d\u00fcn b...
14922013-02-01 Birand\u2019\u0131n 33\u2032\u00fcnc\u00fc g\u00fcn\u00fc \u2013 nediyor.com \\nCan D\u00fcndar ve B\u00fclent \u00c7apl\u0131\u2019n\u0131n 17 Ocak\u2019ta ya...
14792013-02-01 ABD B\u00fcy\u00fckel\u00e7ili\u011fi\u2019ne intihar sald\u0131r\u0131s\u0131 \u2013 nediy... \\nABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nde saat 13:00 ...
14762013-02-01 Sald\u0131r\u0131da talihsiz randevu \u2013 nediyor.com \\n ABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nde ger\u00e7ekle\u015fe...
52 2013-02-01 Ba\u015fbakan: \u201cTSK bir ter\u00f6r \u00f6rg\u00fct\u00fc de\u011fildir\u201d \u2013 ne... \\nBa\u015fbakan Erdo\u011fan, silahl\u0131 ter\u00f6r \u00f6rg\u00fct\u00fc lider...
14642013-02-02 Radikal\u2019e \u201cKorsakoff\u201d tepkisi \u2013 nediyor.com \\nD\u00fcn Ankara\u2019da ABD Ba\u015fkonsoloslu\u011fu\u2019na d\u00fczenle...
14362013-02-04 Sarai\u2019nin soru\u015fturmas\u0131na FBI da dahil oldu \u2013 n... \\n \u0130stanbul\u2019da kaybolduktan bir s\u00fcre sonra Cum...
49 2013-02-05 Didem Tuncay\u2019dan \u00fczen haber \u2013 nediyor.com \\nABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi giri\u015findeki pat...
14192013-02-05 Tarantino\u2019nun \u201cZincirsiz\u201di beklenen ilgiyi g\u00f6r... \\n\u201cEn \u0130yi Film\u201d dahil 5 dalda Oscar (Sinema Sa...
14172013-02-05 Erdo\u011fan \u201c\u015eangay Be\u015flisi\u201dni savundu \u2013 nediyor.com \\nBa\u015fbakan Erdo\u011fan ile Rusya Devlet Ba\u015fkan\u0131 Pu...
45 2013-02-06 Ergin Saygun\u2019un k\u0131z\u0131n\u0131n \u00e7\u0131\u011fl\u0131\u011f\u0131 \u2013 nediyor.com \\nBalyoz davas\u0131nda 18 y\u0131l hapis cezas\u0131 verilen...
13882013-02-06 Ahmedinejad\u2019a da ayakkab\u0131 \u2013 nediyor.com \\n\u0130ran Cumhurba\u015fkan\u0131 Ahmedinejad, Kahire\u2019de Su...
13652013-02-07 FBI Sarai Sierra\u2019dan \u015f\u00fcphelendi \u2013 nediyor.com \\nSarai Sierra\u2019n\u0131n \u00f6l\u00fcm\u00fcn\u00fc soru\u015fturan Amerikan...
13642013-02-07 T\u00fcrkiye uyard\u0131; Obama y\u00f6netimi sahip \u00e7\u0131kt\u0131 \u2013 n... \\nABD B\u00fcy\u00fckel\u00e7isi Francis Ricciardone, T\u00fcrk ya...
13332013-02-08 DHKP-C g\u00f6z\u00fcn\u00fc \u00e7\u0131kartt\u0131; \u00f6z\u00fcr diledi \u2013 nediyor.com \\nDHKP-C, ABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019ne y\u00f6ne...
13352013-02-08 Saygun tahliyesinde Ar\u0131n\u00e7 tesad\u00fcf\u00fc \u2013 nediyor.com \\nBa\u015fbakan Yard\u0131mc\u0131s\u0131 B\u00fclent Ar\u0131n\u00e7, d\u00fcn ak\u015fam ...
............
99442014-11-29 Kobani\u2019deki \u00e7at\u0131\u015fmalar s\u0131n\u0131ra 25 metrede \\nSuriye\u2019de\u00a0Halep\u2019e ba\u011fl\u0131 Kobani\u2019de (Ayn el Ar...
90692014-12-04 ABD B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nden Gaziantep\u2019te sald\u0131r\u0131 uy... \\nABD Ankara B\u00fcy\u00fckel\u00e7ili\u011fi, radikal gruplar\u0131n ...
88572014-12-04 \u201cAK Saray\u201d i\u00e7in a\u00e7\u0131lan davada Atat\u00fcrk\u2019\u00fcn vasiy... \\nAralar\u0131nda Mimarlar Odas\u0131\u2019n\u0131n da bulundu\u011fu m...
89602014-12-04 \u015ei\u015fli Belediyesi\u2019nde \u0130n\u00f6n\u00fc-Sar\u0131g\u00fcl gerilimi \\n\u015ei\u015fli Belediye Ba\u015fkan\u0131 Hayri \u0130n\u00f6n\u00fc, Ba\u015fkan Y...
90682014-12-04 ABD askerlerinin ba\u015f\u0131na \u00e7uval ge\u00e7irenlere 10 y... \\n\u0130stanbul Emin\u00f6n\u00fc semtinde iki ABD askerinin ...
86562014-12-05 T\u00fcrkiye\u2019nin ilk K\u00fclt\u00fcr Bakan\u0131 Halman vefat etti \\nT\u00fcrkiye\u2019nin ilk K\u00fclt\u00fcr Bakan\u0131 ve Bilkent \u00dcni...
87472014-12-05 Tatl\u0131ses: \u201cSaray milletin; bunu g\u00f6remediniz\u201d \\nCumhurba\u015fkanl\u0131\u011f\u0131 Saray\u0131 ile ilgili tart\u0131\u015fmal...
85792014-12-06 Cumhurba\u015fkan\u0131: \u201cPolis kendisini savunmayacak m\u0131?\u201d \\nCumhurba\u015fkan\u0131 Erdo\u011fan, ABD\u2019de son d\u00f6nemde ta...
84862014-12-07 Russell Crowe: \u201cAvustralya\u2019y\u0131 M\u00fcsl\u00fcmanlar ke\u015ff... \\nHollywood\u2019un \u00fcnl\u00fc akt\u00f6rlerinden Russell Crow...
71822014-12-12 Dolar iki ay\u0131n zirvesinde; 2.30 TL\u2019yi ge\u00e7ti \\nDolar, iki ay\u0131n ard\u0131ndan yeniden 2.30 TL sev...
70692014-12-13 \u201cABD\u2019li Ba\u015fkonsolos, T\u00fcrk genciyle evleniyor\u201d ... \\nABD\u2019nin \u0130stanbul Ba\u015fkonsolosu Charles F. Hun...
68622014-12-14 CHP\u2019li Haluk Ko\u00e7\u2019tan yeni \u201ctorpil\u201d listesi \\nHaberListe\\n\\nCHP Genel Ba\u015fkan Yard\u0131mc\u0131s\u0131 ve...
65702014-12-15 Dolarda tarihi rekor; 2.40 TL\u2019ye dayand\u0131 \\n16-17 Aral\u0131k\u2019ta ger\u00e7ekle\u015fecek Amerikan Merke...
62032014-12-16 \u201cDigit\u00fcrk Katar \u015feyhine sat\u0131ld\u0131\u201d iddias\u0131 \\nY\u00fczde 53\u2032\u00fc ABD\u2019li Providence fonuna, y\u00fczde 4...
60952014-12-17 Erdo\u011fan\u2019dan G\u00fclen\u2019e: \u201cElini kolunu ba\u011flayan yo... \\nHaberA\u00e7\u0131klamadan sat\u0131rba\u015flar\u0131\\nKonya-\u0130stanbu...
59082014-12-18 Nazl\u0131 \u0130n\u00f6n\u00fc su\u00e7lad\u0131; Sar\u0131g\u00fcl yalanlad\u0131 \\n\u0130stanbul\u2019da CHP\u2019li \u015ei\u015fli Belediyesi\u2019nde ya\u015fa...
54082014-12-19 Fuat Avni ile G\u00fclerce yeniden kar\u015f\u0131 kar\u015f\u0131ya \\nHaberFuat Avni'nin payla\u015f\u0131mlar\u0131G\u00fclerce'nin y...
54112014-12-19 Fethullah G\u00fclen hakk\u0131nda yakalama talebi \\n14 Aral\u0131k soru\u015fturmas\u0131 kapsam\u0131nda\u00a0Fethullah ...
51222014-12-22 Bula\u00e7 Dilipak\u2019\u0131 do\u011frulad\u0131: \u201cAK Parti bir Ameri... \\nHaber\u0130ddia ve Dilipak'\u0131n yan\u0131t\u0131\\n\\nZaman gaz...
47152014-12-23 Rekt\u00f6r: \u201cTayt ya da mini etekle \u00fcniversiteye g... \\n\u0130stanbul Ayd\u0131n \u00dcniversitesi Rekt\u00f6r\u00fc Prof. Dr...
40102014-12-26 Davuto\u011flu: \u201c\u2018D\u00f6nemin ba\u015fbakan\u0131\u2019 dedikleri ki\u015fi\u2026\u201d \\nBa\u015fbakan Ahmet Davuto\u011flu, ATV ve A Haber kan...
38182014-12-27 B\u00fclent Ar\u0131n\u00e7: \u201cArkadan vurulduk\u201d \\nH\u00fck\u00fcmet S\u00f6zc\u00fcs\u00fc ve Ba\u015fbakan Yard\u0131mc\u0131s\u0131 B\u00fclen...
35212014-12-29 T\u00fcrk ekonomisi iki basamak geriledi \\nUluslararas\u0131 ara\u015ft\u0131rma \u015firketi \u0130ngiliz Ekono...
36242014-12-29 G\u00fclen\u2019in pasaportunun iptaline Dan\u0131\u015ftay vizesi \\nHaber\u015eimdi ne olacak?\\n\\nDan\u0131\u015ftay 10\u2032uncu Da...
30622014-12-31 T\u00fcrkiye\u2019de 2014 b\u00f6yle ge\u00e7ti \\nHaberAy ay 2014'\u00fcn g\u00f6r\u00fcn\u00fcm\u00fc\\n\\nT\u00fcrkiye harek...
24882015-01-02 MEB\u2019den yeni \u201cde\u011ferler e\u011fitimi\u201d \\nHaberKitap\u00e7\u0131ktan b\u00f6l\u00fcmler\\nMilli E\u011fitim Baka...
24862015-01-03 Dumanl\u0131\u2019dan ABD bas\u0131n\u0131nda a\u011f\u0131r Erdo\u011fan ele\u015ftirisi \\nZaman gazetesi Genel Yay\u0131n Y\u00f6netmeni\u00a0Ekrem D...
22732015-01-04 Tatl\u0131ses milletvekili adayl\u0131\u011f\u0131n\u0131 duyurdu \\n\u00dcnl\u00fc t\u00fcrk\u00fcc\u00fc \u0130brahim Tatl\u0131ses, 2015 se\u00e7imler...
10942015-01-07 \u00d6DP lideri Ta\u015f: \u201cT\u00fcrkiye\u2019de \u2018devrimci ilahiyat... \\nH\u00fcrriyet gazetesinde Ahmet Hakan\u2019\u0131n bu hafta...
176 2015-01-08 Kadri G\u00fcrsel: \u201cFransa\u2019n\u0131n Mad\u0131mak\u2019\u0131\u201d \\nMilliyet gazetesi yazar\u0131 Kadri G\u00fcrsel,\u00a0Frans...
\n", "

585 rows \u00d7 3 columns

\n", "
" ], "metadata": {}, "output_type": "pyout", "prompt_number": 22, "text": [ " dt title \\\n", "1916 2013-01-14 1979 \u0130ran\u2019\u0131na \u00f6d\u00fcl ya\u011fd\u0131 \u2013 nediyor.com \n", "1917 2013-01-14 Sa\u011fl\u0131k Bakanl\u0131\u011f\u0131\u2019ndan Refiye a\u00e7\u0131klamas\u0131 \u2013 nedi... \n", "1852 2013-01-16 \u00dcnl\u00fc ressam Do\u011fan\u00e7ay hayat\u0131n\u0131 kaybetti \u2013 nediy... \n", "1851 2013-01-16 Mehmet Ali Birand hastaneye kald\u0131r\u0131ld\u0131 \u2013 nediy... \n", "1812 2013-01-17 Gazetecili\u011fin ikonu Birand\u2019\u0131 kaybettik \u2013 nediy... \n", "1825 2013-01-17 \u00d6l\u00fcm haberine Umur Birand\u2019tan yalanlama \u2013 nedi... \n", "1795 2013-01-18 Birand\u2019\u0131n \u00f6l\u00fcm\u00fcnde ihmal \u015f\u00fcphesi \u2013 nediyor.com \n", "1796 2013-01-18 Armstrong sonunda dopingi itiraf etti \u2013 nediyo... \n", "1788 2013-01-19 Birand son yolculu\u011funa u\u011furland\u0131 \u2013 nediyor.com \n", "1761 2013-01-21 Patriotlar T\u00fcrkiye\u2019de \u2013 nediyor.com \n", "70 2013-01-22 Hatay\u2019da NATO askerine \u00e7uval ge\u00e7irme giri\u015fimi ... \n", "1744 2013-01-22 Obama ikinci kez yemin etti \u2013 nediyor.com \n", "1647 2013-01-25 Samatya\u2019daki sald\u0131r\u0131lar\u0131n faili ayn\u0131 ki\u015fi \u2013 ne... \n", "1551 2013-01-29 \u201c\u015eangay Be\u015flisi\u201d a\u00e7\u0131l\u0131m\u0131na ABD\u2019den g\u00fcl\u00fcmseme \u2013... \n", "56 2013-01-30 \u0130srail u\u00e7aklar\u0131 Suriye\u2019yi vurdu \u2013 nediyor.com \n", "1492 2013-02-01 Birand\u2019\u0131n 33\u2032\u00fcnc\u00fc g\u00fcn\u00fc \u2013 nediyor.com \n", "1479 2013-02-01 ABD B\u00fcy\u00fckel\u00e7ili\u011fi\u2019ne intihar sald\u0131r\u0131s\u0131 \u2013 nediy... \n", "1476 2013-02-01 Sald\u0131r\u0131da talihsiz randevu \u2013 nediyor.com \n", "52 2013-02-01 Ba\u015fbakan: \u201cTSK bir ter\u00f6r \u00f6rg\u00fct\u00fc de\u011fildir\u201d \u2013 ne... \n", "1464 2013-02-02 Radikal\u2019e \u201cKorsakoff\u201d tepkisi \u2013 nediyor.com \n", "1436 2013-02-04 Sarai\u2019nin soru\u015fturmas\u0131na FBI da dahil oldu \u2013 n... \n", "49 2013-02-05 Didem Tuncay\u2019dan \u00fczen haber \u2013 nediyor.com \n", "1419 2013-02-05 Tarantino\u2019nun \u201cZincirsiz\u201di beklenen ilgiyi g\u00f6r... \n", "1417 2013-02-05 Erdo\u011fan \u201c\u015eangay Be\u015flisi\u201dni savundu \u2013 nediyor.com \n", "45 2013-02-06 Ergin Saygun\u2019un k\u0131z\u0131n\u0131n \u00e7\u0131\u011fl\u0131\u011f\u0131 \u2013 nediyor.com \n", "1388 2013-02-06 Ahmedinejad\u2019a da ayakkab\u0131 \u2013 nediyor.com \n", "1365 2013-02-07 FBI Sarai Sierra\u2019dan \u015f\u00fcphelendi \u2013 nediyor.com \n", "1364 2013-02-07 T\u00fcrkiye uyard\u0131; Obama y\u00f6netimi sahip \u00e7\u0131kt\u0131 \u2013 n... \n", "1333 2013-02-08 DHKP-C g\u00f6z\u00fcn\u00fc \u00e7\u0131kartt\u0131; \u00f6z\u00fcr diledi \u2013 nediyor.com \n", "1335 2013-02-08 Saygun tahliyesinde Ar\u0131n\u00e7 tesad\u00fcf\u00fc \u2013 nediyor.com \n", "... ... ... \n", "9944 2014-11-29 Kobani\u2019deki \u00e7at\u0131\u015fmalar s\u0131n\u0131ra 25 metrede \n", "9069 2014-12-04 ABD B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nden Gaziantep\u2019te sald\u0131r\u0131 uy... \n", "8857 2014-12-04 \u201cAK Saray\u201d i\u00e7in a\u00e7\u0131lan davada Atat\u00fcrk\u2019\u00fcn vasiy... \n", "8960 2014-12-04 \u015ei\u015fli Belediyesi\u2019nde \u0130n\u00f6n\u00fc-Sar\u0131g\u00fcl gerilimi \n", "9068 2014-12-04 ABD askerlerinin ba\u015f\u0131na \u00e7uval ge\u00e7irenlere 10 y... \n", "8656 2014-12-05 T\u00fcrkiye\u2019nin ilk K\u00fclt\u00fcr Bakan\u0131 Halman vefat etti \n", "8747 2014-12-05 Tatl\u0131ses: \u201cSaray milletin; bunu g\u00f6remediniz\u201d \n", "8579 2014-12-06 Cumhurba\u015fkan\u0131: \u201cPolis kendisini savunmayacak m\u0131?\u201d \n", "8486 2014-12-07 Russell Crowe: \u201cAvustralya\u2019y\u0131 M\u00fcsl\u00fcmanlar ke\u015ff... \n", "7182 2014-12-12 Dolar iki ay\u0131n zirvesinde; 2.30 TL\u2019yi ge\u00e7ti \n", "7069 2014-12-13 \u201cABD\u2019li Ba\u015fkonsolos, T\u00fcrk genciyle evleniyor\u201d ... \n", "6862 2014-12-14 CHP\u2019li Haluk Ko\u00e7\u2019tan yeni \u201ctorpil\u201d listesi \n", "6570 2014-12-15 Dolarda tarihi rekor; 2.40 TL\u2019ye dayand\u0131 \n", "6203 2014-12-16 \u201cDigit\u00fcrk Katar \u015feyhine sat\u0131ld\u0131\u201d iddias\u0131 \n", "6095 2014-12-17 Erdo\u011fan\u2019dan G\u00fclen\u2019e: \u201cElini kolunu ba\u011flayan yo... \n", "5908 2014-12-18 Nazl\u0131 \u0130n\u00f6n\u00fc su\u00e7lad\u0131; Sar\u0131g\u00fcl yalanlad\u0131 \n", "5408 2014-12-19 Fuat Avni ile G\u00fclerce yeniden kar\u015f\u0131 kar\u015f\u0131ya \n", "5411 2014-12-19 Fethullah G\u00fclen hakk\u0131nda yakalama talebi \n", "5122 2014-12-22 Bula\u00e7 Dilipak\u2019\u0131 do\u011frulad\u0131: \u201cAK Parti bir Ameri... \n", "4715 2014-12-23 Rekt\u00f6r: \u201cTayt ya da mini etekle \u00fcniversiteye g... \n", "4010 2014-12-26 Davuto\u011flu: \u201c\u2018D\u00f6nemin ba\u015fbakan\u0131\u2019 dedikleri ki\u015fi\u2026\u201d \n", "3818 2014-12-27 B\u00fclent Ar\u0131n\u00e7: \u201cArkadan vurulduk\u201d \n", "3521 2014-12-29 T\u00fcrk ekonomisi iki basamak geriledi \n", "3624 2014-12-29 G\u00fclen\u2019in pasaportunun iptaline Dan\u0131\u015ftay vizesi \n", "3062 2014-12-31 T\u00fcrkiye\u2019de 2014 b\u00f6yle ge\u00e7ti \n", "2488 2015-01-02 MEB\u2019den yeni \u201cde\u011ferler e\u011fitimi\u201d \n", "2486 2015-01-03 Dumanl\u0131\u2019dan ABD bas\u0131n\u0131nda a\u011f\u0131r Erdo\u011fan ele\u015ftirisi \n", "2273 2015-01-04 Tatl\u0131ses milletvekili adayl\u0131\u011f\u0131n\u0131 duyurdu \n", "1094 2015-01-07 \u00d6DP lideri Ta\u015f: \u201cT\u00fcrkiye\u2019de \u2018devrimci ilahiyat... \n", "176 2015-01-08 Kadri G\u00fcrsel: \u201cFransa\u2019n\u0131n Mad\u0131mak\u2019\u0131\u201d \n", "\n", " newstxt \n", "1916 \\nOscar\u2019dan sonra sinema d\u00fcnyas\u0131n\u0131n en prestij... \n", "1917 \\nMilyonda bir g\u00f6r\u00fclen \u201cGardner sendromu\u201d hast... \n", "1852 \\n\u00dcnl\u00fc ressam Burhan Do\u011fan\u00e7ay bu sabah tedavi ... \n", "1851 \\nKanal D Haber Grup Ba\u015fkan\u0131 ve anchorman\u2019\u0131 de... \n", "1812 \\nKanal D Haber Grup Ba\u015fkan\u0131 Mehmet Ali Birand... \n", "1825 \\nKanal D Haber Grup Ba\u015fkan\u0131 Mehmet Ali Birand... \n", "1795 \\nDuayen gazeteci Mehmet Ali Birand\u2019\u0131n Sal\u0131 g\u00fc... \n", "1796 \\nAmerika\u2019n\u0131n g\u00fcndemi Lance Armstrong. \u00dcnl\u00fc bi... \n", "1788 \\nPer\u015fembe g\u00fcn\u00fc vefat eden duayen gazeteci Meh... \n", "1761 \\nT\u00fcrkiye\u2019nin Suriye s\u0131n\u0131r\u0131na konu\u015fland\u0131rmak \u00fc... \n", "70 \\nHatay\u2019\u0131n \u0130skenderun il\u00e7esinde T\u00fcrkiye Gen\u00e7li... \n", "1744 \\nABD Ba\u015fkan\u0131 Barack Obama, ikinci ba\u015fkanl\u0131k d... \n", "1647 \\n\u0130stanbul\u2019da ge\u00e7ti\u011fimiz Aral\u0131k ay\u0131ndan bu yan... \n", "1551 \\nBa\u015fbakan Erdo\u011fan\u2019\u0131n AB\u2019ye alternatif olarak ... \n", "56 \\nSuriye, \u0130srail\u2019e ait sava\u015f u\u00e7aklar\u0131n\u0131n d\u00fcn b... \n", "1492 \\nCan D\u00fcndar ve B\u00fclent \u00c7apl\u0131\u2019n\u0131n 17 Ocak\u2019ta ya... \n", "1479 \\nABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nde saat 13:00 ... \n", "1476 \\n ABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019nde ger\u00e7ekle\u015fe... \n", "52 \\nBa\u015fbakan Erdo\u011fan, silahl\u0131 ter\u00f6r \u00f6rg\u00fct\u00fc lider... \n", "1464 \\nD\u00fcn Ankara\u2019da ABD Ba\u015fkonsoloslu\u011fu\u2019na d\u00fczenle... \n", "1436 \\n \u0130stanbul\u2019da kaybolduktan bir s\u00fcre sonra Cum... \n", "49 \\nABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi giri\u015findeki pat... \n", "1419 \\n\u201cEn \u0130yi Film\u201d dahil 5 dalda Oscar (Sinema Sa... \n", "1417 \\nBa\u015fbakan Erdo\u011fan ile Rusya Devlet Ba\u015fkan\u0131 Pu... \n", "45 \\nBalyoz davas\u0131nda 18 y\u0131l hapis cezas\u0131 verilen... \n", "1388 \\n\u0130ran Cumhurba\u015fkan\u0131 Ahmedinejad, Kahire\u2019de Su... \n", "1365 \\nSarai Sierra\u2019n\u0131n \u00f6l\u00fcm\u00fcn\u00fc soru\u015fturan Amerikan... \n", "1364 \\nABD B\u00fcy\u00fckel\u00e7isi Francis Ricciardone, T\u00fcrk ya... \n", "1333 \\nDHKP-C, ABD\u2019nin Ankara B\u00fcy\u00fckel\u00e7ili\u011fi\u2019ne y\u00f6ne... \n", "1335 \\nBa\u015fbakan Yard\u0131mc\u0131s\u0131 B\u00fclent Ar\u0131n\u00e7, d\u00fcn ak\u015fam ... \n", "... ... \n", "9944 \\nSuriye\u2019de\u00a0Halep\u2019e ba\u011fl\u0131 Kobani\u2019de (Ayn el Ar... \n", "9069 \\nABD Ankara B\u00fcy\u00fckel\u00e7ili\u011fi, radikal gruplar\u0131n ... \n", "8857 \\nAralar\u0131nda Mimarlar Odas\u0131\u2019n\u0131n da bulundu\u011fu m... \n", "8960 \\n\u015ei\u015fli Belediye Ba\u015fkan\u0131 Hayri \u0130n\u00f6n\u00fc, Ba\u015fkan Y... \n", "9068 \\n\u0130stanbul Emin\u00f6n\u00fc semtinde iki ABD askerinin ... \n", "8656 \\nT\u00fcrkiye\u2019nin ilk K\u00fclt\u00fcr Bakan\u0131 ve Bilkent \u00dcni... \n", "8747 \\nCumhurba\u015fkanl\u0131\u011f\u0131 Saray\u0131 ile ilgili tart\u0131\u015fmal... \n", "8579 \\nCumhurba\u015fkan\u0131 Erdo\u011fan, ABD\u2019de son d\u00f6nemde ta... \n", "8486 \\nHollywood\u2019un \u00fcnl\u00fc akt\u00f6rlerinden Russell Crow... \n", "7182 \\nDolar, iki ay\u0131n ard\u0131ndan yeniden 2.30 TL sev... \n", "7069 \\nABD\u2019nin \u0130stanbul Ba\u015fkonsolosu Charles F. Hun... \n", "6862 \\nHaberListe\\n\\nCHP Genel Ba\u015fkan Yard\u0131mc\u0131s\u0131 ve... \n", "6570 \\n16-17 Aral\u0131k\u2019ta ger\u00e7ekle\u015fecek Amerikan Merke... \n", "6203 \\nY\u00fczde 53\u2032\u00fc ABD\u2019li Providence fonuna, y\u00fczde 4... \n", "6095 \\nHaberA\u00e7\u0131klamadan sat\u0131rba\u015flar\u0131\\nKonya-\u0130stanbu... \n", "5908 \\n\u0130stanbul\u2019da CHP\u2019li \u015ei\u015fli Belediyesi\u2019nde ya\u015fa... \n", "5408 \\nHaberFuat Avni'nin payla\u015f\u0131mlar\u0131G\u00fclerce'nin y... \n", "5411 \\n14 Aral\u0131k soru\u015fturmas\u0131 kapsam\u0131nda\u00a0Fethullah ... \n", "5122 \\nHaber\u0130ddia ve Dilipak'\u0131n yan\u0131t\u0131\\n\\nZaman gaz... \n", "4715 \\n\u0130stanbul Ayd\u0131n \u00dcniversitesi Rekt\u00f6r\u00fc Prof. Dr... \n", "4010 \\nBa\u015fbakan Ahmet Davuto\u011flu, ATV ve A Haber kan... \n", "3818 \\nH\u00fck\u00fcmet S\u00f6zc\u00fcs\u00fc ve Ba\u015fbakan Yard\u0131mc\u0131s\u0131 B\u00fclen... \n", "3521 \\nUluslararas\u0131 ara\u015ft\u0131rma \u015firketi \u0130ngiliz Ekono... \n", "3624 \\nHaber\u015eimdi ne olacak?\\n\\nDan\u0131\u015ftay 10\u2032uncu Da... \n", "3062 \\nHaberAy ay 2014'\u00fcn g\u00f6r\u00fcn\u00fcm\u00fc\\n\\nT\u00fcrkiye harek... \n", "2488 \\nHaberKitap\u00e7\u0131ktan b\u00f6l\u00fcmler\\nMilli E\u011fitim Baka... \n", "2486 \\nZaman gazetesi Genel Yay\u0131n Y\u00f6netmeni\u00a0Ekrem D... \n", "2273 \\n\u00dcnl\u00fc t\u00fcrk\u00fcc\u00fc \u0130brahim Tatl\u0131ses, 2015 se\u00e7imler... \n", "1094 \\nH\u00fcrriyet gazetesinde Ahmet Hakan\u2019\u0131n bu hafta... \n", "176 \\nMilliyet gazetesi yazar\u0131 Kadri G\u00fcrsel,\u00a0Frans... \n", "\n", "[585 rows x 3 columns]" ] } ], "prompt_number": 22 }, { "cell_type": "code", "collapsed": false, "input": [ "stemmer = snowballstemmer.TurkishStemmer()\n", "stoplist = stopwords.words('turkish')\n", "stoplist.extend(\"active number yes titlesection bootstrap tabtitle tboot tab tabcontent contentsection nin n\u0131n bir\".split())\n", "stoplist = stemmer.stemWords(stoplist)\n", "stoplist = set(stoplist)" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 4 }, { "cell_type": "code", "collapsed": false, "input": [ "texts=[]\n", "for doc in documents:\n", " doc = re.sub('[!\"#%\\'()*+,-./:;<=>?@\\[\\]^_`{|}~1234567890\u2019\u201d\u201c\u2032\u2018\\\\\\]',' ', doc).split()\n", " doc = stemmer.stemWords(doc)\n", " text= [w.strip() for w in doc if len(w.strip())>=3 and w.strip() not in stoplist]\n", " texts.append(text)" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "dictionary = corpora.Dictionary(texts)\n", "corpus = [dictionary.doc2bow(text) for text in texts]\n", "tfidf = models.TfidfModel(corpus)\n", "corpus_tfidf = tfidf[corpus]" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 6 }, { "cell_type": "code", "collapsed": false, "input": [ "n_topics = 10\n", "lda = models.LdaModel(corpus_tfidf, id2word=dictionary, num_topics=n_topics, passes=5,iterations=500)\n", "for i in range(0, n_topics):\n", " terms = [term[1] for term in lda.show_topic(i, 10)]\n", " print (\"#\" + str(i) + \": \"+ \", \".join(terms))" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stderr", "text": [ "WARNING:gensim.models.ldamodel:too few updates, training might not converge; consider increasing the number of passes or iterations to improve accuracy\n" ] }, { "output_type": "stream", "stream": "stdout", "text": [ "#0: I\u015e\u0130D, Davuto\u011flu, \u00f6d\u00fcl, Baka, \u0130srail, Akdo\u011fa, Seba, Kerry, Mad\u0131mak, sonra\n", "#1: Gez, Sevgili, de\u011fer, iPho, fiyat, Pap, Erdo\u011fa, Apple, Metiner, logo\n", "#2: seviye, Bankas, milyar, puan, rekor, Merkez, rapor, i\u015fle, Bor, lira\n", "#3: Kotil, Bankas, AVM, iptal, THY, Suri, Meclis, pasaport, Davuto\u011flu, FED\n", "#4: Birand\u0131, Menderes, Twitter, \u00fcnl\u00fc, m\u00fcdahale, leopar, eri\u015f, G\u00fcle, Erdo\u011fa, oyun\n", "#5: G\u00fcle, ceket, Ricciardo, Kerk\u00fck, Twitter, Zenca, MERS, \u0130srail, Ba\u015fbaka, \u00c7et\n", "#6: Gez, Davuto\u011flu, Ba\u015fbaka, G\u00fcle, Yerkel, Holla, patriot, Be\u015flis, Halma, May\u0131s\n", "#7: Nusra, Gazze, Dilipak, ki\u015f, ALS, ba\u011f\u0131\u015f, ziyaret, haber, Savaro, silah\n", "#8: CHP, Erdo\u011fa, milyar, haber, G\u00f6k\u00e7ek, Genel, T\u00fcrki, Amerika, yer, y\u0131l\n", "#9: G\u00fcle, Erdo\u011fa, I\u015e\u0130D, Ba\u015fbaka, T\u00fcrki, Kobani, Suri, dedi, Oba, konu\n" ] } ], "prompt_number": 7 }, { "cell_type": "code", "collapsed": false, "input": [ "i = 0\n", "print ('Topic distribution on an arbitrarily selected document:\\n')\n", "print ('Original document: ' + documents[i])\n", "print ('Preprocessed document: ' + str(texts[i]))" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "Topic distribution on an arbitrarily selected document:\n", "\n", "Original document: \n", "ABD Ba\u015fkan\u0131 Barack Obama ile 16 May\u0131s\u2019ta g\u00f6r\u00fc\u015fmek i\u00e7in Amerika\u2019ya giden Ba\u015fbakan Erdo\u011fan, bir gazetecinin \u201cFethullah G\u00fclen\u2019le g\u00f6r\u00fc\u015fme ihtimaliniz olabilir mi?\u201d sorusuna, \u201c\u015eu anda resmi program\u0131m\u0131zda b\u00f6yle bir \u015fey yok ama g\u00f6kten ne ya\u011far ki yer kabul etmez ayr\u0131 bir konu\u201d diyerek cevaplad\u0131.\n", "1999 y\u0131l\u0131nda ABD\u2019nin Pensilvanya eyaletine yerle\u015fen G\u00fclen, ge\u00e7ti\u011fimiz ay ABD\u2019li Time dergisi taraf\u0131ndan Abdullah \u00d6calan ile birlikte \u201cD\u00fcnyan\u0131n en etkili 100 ismi\u201d aras\u0131nda g\u00f6sterilmi\u015fti. G\u00fclen, ge\u00e7ti\u011fimiz haftalarda, aralar\u0131nda Ekrem Dumanl\u0131, Mehmet Altan, Mahmut \u00d6v\u00fcr ve Amberin Zaman\u2019\u0131n da oldu\u011fu baz\u0131 gazetecilerle ABD\u2019de g\u00f6r\u00fc\u015fm\u00fc\u015ft\u00fc. G\u00f6r\u00fc\u015fmede, G\u00fclen\u2019in\u00a0AK Parti h\u00fck\u00fcmetini ele\u015ftirdi\u011fi y\u00f6n\u00fcnde iddialar bas\u0131na yans\u0131m\u0131\u015ft\u0131.\n", "Ba\u015fbakan Erdo\u011fan\u2019da ge\u00e7ti\u011fimiz Haziran ay\u0131nda T\u00fcrk\u00e7e Olimpiyatlar\u0131 final t\u00f6reninde yapt\u0131\u011f\u0131 konu\u015fmada G\u00fclen\u2019e \u201cBitsin bu gurbet, bitsin bu hasret\u2026\u201c diye seslenmi\u015fti.\n", "\n", "Preprocessed document: ['ABD', 'Ba\u015fka', 'Barack', 'Oba', 'May\u0131s', 'g\u00f6r\u00fc\u015fmek', 'Amerika', 'Ba\u015fbaka', 'Erdo\u011fa', 'gazete\u00e7', 'Fethullah', 'G\u00fcle', 'g\u00f6r\u00fc\u015fme', 'ihtimal', 'olabilir', 'soru', 'resmi', 'program', 'yok', 'g\u00f6k', 'ya\u011far', 'yer', 'kabul', 'etmez', 'ayr\u0131', 'diyerek', 'cevaplad\u0131', 'y\u0131l', 'ABD', 'Pensilvanya', 'eyalet', 'yerle\u015fe', 'G\u00fcle', 'ge\u00e7tik', 'ABD', 'dergis', 'taraf', 'Abdullah', '\u00d6cala', 'birlik', 'D\u00fcnya', 'etkil', 'ismi', 'ara', 'g\u00f6sterilmi\u015f', 'G\u00fcle', 'ge\u00e7tik', 'hafta', 'ara', 'Ekrem', 'Dumanl\u0131', 'Mehmet', 'Altan', 'Mahmut', '\u00d6v\u00fcr', 'Amber', 'Zama', 'olduk', 'gazete\u00e7', 'ABD', 'g\u00f6r\u00fc\u015fm\u00fc\u015f', 'G\u00f6r\u00fc\u015fme', 'G\u00fcle', 'Parti', 'h\u00fck\u00fcmet', 'ele\u015ftirdik', 'y\u00f6n', 'iddia', 'yans\u0131m\u0131\u015f', 'Ba\u015fbaka', 'Erdo\u011fa', 'ge\u00e7tik', 'Hazira', 'T\u00fcrk\u00e7e', 'Olimpiyat', 'final', 't\u00f6ren', 'yapt\u0131k', 'konu\u015fma', 'G\u00fcle', 'Bit', 'gurbet', 'bit', 'hasret\u2026', 'seslenmi\u015f']\n" ] } ], "prompt_number": 8 }, { "cell_type": "code", "collapsed": false, "input": [ "print ('Which LDA topic maximally describes this document?')\n", "print ('Topic probability mixture: ',end=\"\")\n", "for j,k in lda[corpus[i]]:\n", " print('(%d, %3.2f)' % (j,k), end = ' ')\n", "print ('\\nTopic with the highest probability: topic #' + str(max(lda[corpus[i]],key=itemgetter(1))[0]))" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "Which LDA topic maximally describes this document?\n", "Topic probability mixture: (6, 0.31) (9, 0.68) \n", "Topic with the highest probability: topic #9\n" ] } ], "prompt_number": 9 } ], "metadata": {} } ] }