{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2016-08-29T16:14:30.433874",
"start_time": "2016-08-29T16:14:30.429136"
}
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import matplotlib as mpl"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 0. 数据清洗"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"bigfile = open('/Users/chengjun/百度云同步盘/Writing/OWS/ows-raw.txt', 'rb')\n",
"chunkSize = 10000000\n",
"chunk = bigfile.readlines(chunkSize)\n",
"block = ''\n",
"with open(\"/Users/chengjun/百度云同步盘/Writing/OWS/ows-clean.txt\", 'w') as f:\n",
" while chunk:\n",
" blocks = []\n",
" for line in chunk:\n",
" block += line.replace('\\n', '')\n",
" block_length = len(block.split(','))\n",
" if block_length >= 14:\n",
" blocks.append(block)\n",
" block = ''\n",
" for i in blocks:\n",
" f.write(i + '\\n')\n",
" chunk = bigfile.readlines(chunkSize)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 1. 数据总量"
]
},
{
"cell_type": "code",
"execution_count": 120,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6602141 6602186 21\n"
]
}
],
"source": [
"from collections import defaultdict\n",
"import csv\n",
"\n",
"data_dict = defaultdict(list)\n",
"\n",
"error_num = 0\n",
"line_num = 0\n",
"total_num = 0\n",
"\n",
"bigfile = open('/Users/chengjun/百度云同步盘/Writing/OWS/ows-clean.txt', 'rb')\n",
"chunkSize = 100000000\n",
"chunk = bigfile.readlines(chunkSize)\n",
"while chunk:\n",
" total_num += len(chunk)\n",
" lines = csv.reader((line.replace('\\x00','') for line in chunk), delimiter=',', quotechar='\"')\n",
" for i in lines:\n",
" line_num +=1\n",
" try:\n",
" date = i[3]\n",
" if len(date) == 10:\n",
" data_dict[date].append(i[8])\n",
" else:\n",
" error_num+=1\n",
" except:\n",
" pass\n",
" chunk = bigfile.readlines(chunkSize)\n",
"print line_num, total_num, error_num"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### If you can see \\x00 or \\0, then you definitely have NULL bytes in the file, and you need to replace them before using csv.reader http://stackoverflow.com/questions/4166070/python-csv-error-line-contains-null-byte"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### 注释：Error: line contains NULL byte是造成40tweets无法解析的关键, 21个不是日期，共损失60条tweets。"
]
},
{
"cell_type": "code",
"execution_count": 121,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'2012-01-23 , 2011-10-25 , 2011-10-24 , 2011-12-10 , 2011-12-11 , 2011-12-12 , 2011-12-13 , 2011-12-14 , 2011-12-15 , 2011-12-16 , 2011-12-17 , 2011-12-18 , 2011-12-19 , 2012-02-16 , 2012-02-17 , 2012-02-14 , 2012-02-15 , 2012-02-12 , 2012-02-13 , 2012-02-10 , 2012-02-11 , 2012-02-18 , 2011-12-07 , 2011-12-06 , 2011-12-05 , 2011-12-04 , 2011-12-03 , 2011-12-02 , 2011-12-01 , 2011-11-20 , 2011-11-21 , 2011-11-22 , 2011-11-23 , 2011-11-24 , 2011-11-25 , 2011-12-09 , 2011-12-08 , 2012-02-05 , 2012-02-04 , 2012-02-07 , 2012-02-06 , 2012-02-01 , 2012-02-03 , 2012-02-02 , 2012-02-09 , 2012-02-08 , 2011-11-11 , 2011-10-31 , 2011-11-28 , 2012-01-28 , 2011-11-29 , 2012-01-26 , 2012-01-27 , 2012-01-24 , 2012-01-25 , 2012-01-22 , 2011-10-29 , 2012-01-20 , 2012-01-21 , 2011-10-28 , 2011-11-30 , 2011-10-23 , 2011-10-22 , 2011-10-21 , 2011-10-20 , 2011-10-27 , 2012-01-31 , 2012-01-30 , 2011-10-26 , 2011-11-26 , 2011-11-27 , 2012-01-01 , 2012-01-02 , 2012-01-03 , 2012-01-04 , 2012-01-05 , 2012-01-06 , 2012-01-07 , 2012-01-08 , 2012-01-09 , 2011-11-01 , 2011-10-12 , 2011-10-13 , 2011-10-10 , 2011-10-11 , 2011-10-16 , 2011-10-17 , 2011-10-14 , 2011-10-15 , 2011-10-18 , 2011-10-19 , 2012-01-17 , 2012-01-16 , 2012-01-15 , 2012-01-14 , 2012-01-13 , 2012-01-12 , 2012-01-11 , 2012-01-10 , 2012-01-19 , 2012-01-18 , 2011-11-06 , 2011-11-07 , 2011-11-04 , 2011-11-05 , 2011-11-02 , 2011-11-03 , 2011-10-07 , 2011-10-06 , 2011-10-09 , 2011-10-08 , 2011-11-08 , 2011-11-09 , 2011-12-30 , 2011-12-31 , 2011-11-15 , 2011-11-14 , 2011-11-17 , 2011-11-16 , 2011-10-30 , 2011-11-10 , 2011-11-13 , 2011-11-12 , 2011-11-19 , 2011-11-18 , 2012-01-29 , 2011-12-29 , 2011-12-28 , 2011-12-25 , 2011-12-24 , 2011-12-27 , 2011-12-26 , 2011-12-21 , 2011-12-20 , 2011-12-23 , 2011-12-22'"
]
},
"execution_count": 121,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\" , \".join(data_dict.keys())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. 每天的tweets数量"
]
},
{
"cell_type": "code",
"execution_count": 123,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"data = [[i, len(data_dict[i]), len(set(data_dict[i]))] for i in data_dict]\n",
"dat = pd.DataFrame(data, columns = ['date', 'tweets', 'users'])\n",
"dat.date = pd.to_datetime(dat.date)\n",
"dat = dat.sort(['date'])"
]
},
{
"cell_type": "code",
"execution_count": 126,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"6602120"
]
},
"execution_count": 126,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"np.sum(dat.tweets)"
]
},
{
"cell_type": "code",
"execution_count": 127,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" \n",
" | \n",
" tweets | \n",
" users | \n",
"
\n",
" \n",
" \n",
" \n",
" | count | \n",
" 136.000000 | \n",
" 136.000000 | \n",
"
\n",
" \n",
" | mean | \n",
" 48545.000000 | \n",
" 16800.360294 | \n",
"
\n",
" \n",
" | std | \n",
" 51250.621113 | \n",
" 14859.205183 | \n",
"
\n",
" \n",
" | min | \n",
" 4859.000000 | \n",
" 2012.000000 | \n",
"
\n",
" \n",
" | 25% | \n",
" 15803.750000 | \n",
" 5945.500000 | \n",
"
\n",
" \n",
" | 50% | \n",
" 37204.500000 | \n",
" 13153.000000 | \n",
"
\n",
" \n",
" | 75% | \n",
" 64630.250000 | \n",
" 23297.250000 | \n",
"
\n",
" \n",
" | max | \n",
" 409075.000000 | \n",
" 98934.000000 | \n",
"
\n",
" \n",
"
\n",
"
\n",
"
\n",
" \n",
" \n",
" | \n",
" date | \n",
" users | \n",
" tweets | \n",
" mentionUnique | \n",
" mentions | \n",
" retweetUnique | \n",
" retweets | \n",
" urlUnique | \n",
" urls | \n",
" hashtagUnique | \n",
" hashtags | \n",
"
\n",
" \n",
" \n",
" \n",
" | 108 | \n",
" 2011-10-06 | \n",
" 18487 | \n",
" 49638 | \n",
" 7810 | \n",
" 54874 | \n",
" 4969 | \n",
" 30841 | \n",
" 9783 | \n",
" 22807 | \n",
" 4921 | \n",
" 112364 | \n",
"
\n",
" \n",
" | 107 | \n",
" 2011-10-07 | \n",
" 23460 | \n",
" 65238 | \n",
" 9972 | \n",
" 72744 | \n",
" 6493 | \n",
" 39318 | \n",
" 14479 | \n",
" 33632 | \n",
" 6748 | \n",
" 149517 | \n",
"
\n",
" \n",
"
\n",
"