(.*?)<\/p>/'; } $rs = preg_match_all($pattern, $content, $matches); if ($rs) { mylog('crawl finished.start parsing'); $data = array(); foreach ($matches[1] as $key => $val) { if (false !== strpos($matches[1][$key], 'job') || false !== strpos($matches[1][$key], 'amazon')) { // get rid of job and book recommendations continue; } $item = [ 'title' => strip_tags($matches[2][$key]), 'desc' => $matches[3][$key], 'href' => $val, 'number' => $number, 'hash' => md5($val), ]; //加入数据库 $id = $this->add_cache($item); $item['id'] = $id; mylog("item {$id} added to cache"); // 加入数组 $data[] = $item; } return $data; } mylog('failed to crawl'); return false; } /** * add item to cache */ function add_cache($item) { $sql = $this->arr2sql('cache', $item); $this->pdo->exec($sql); $id = $this->pdo->lastInsertId(); return $id; } /** * translate array to sql */ function arr2sql($dbname, $arr, $updateid = null) { if ($updateid) { // return "update {$dbname} set " unset($arr['ctime']); unset($arr['addtime']); $sql = "update {$dbname} set "; foreach ($arr as $key => $val) { $sql .= "`{$key}`='{$val}',"; } $sql .= 'ctime=' . time(); $sql .= " where id={$updateid}"; return $sql; } else { $fields = []; $values = []; foreach ($arr as $key => $v) { $fields[] = "`{$key}`"; $values[] = "'{$v}'"; } return "insert into {$dbname} (" . implode(',', $fields) . ") values(" . implode(',', $values) . ")"; } } /** * get categories */ function get_cate() { $arr = []; $rs = $this->pdo->query("select distinct category from issue order by category")->fetchAll(PDO::FETCH_ASSOC); foreach ($rs as $val) { $arr[] = $val['category']; } return $arr; } /** * add an item to db or update it if exists */ function add($data) { $sql = $this->arr2sql('issue', $data); $rs = $this->pdo->exec($sql); if ($rs) { return $this->pdo->lastInsertId(); } $rs = $this->pdo->query("select id from issue where hash='{$data['hash']}'")->fetch(PDO::FETCH_ASSOC); if (isset($rs['id'])) { $update = $this->pdo->exec($this->arr2sql('issue', $data, $rs['id'])); if ($update) { return $rs['id']; } } return false; } /** * fetch all recorded data */ function get_all() { return $this->pdo->query('select * from issue order by category,addtime')->fetchAll(PDO::FETCH_ASSOC); } function current_number() { $rs = $this->pdo->query('select max(number) as max from issue')->fetch(PDO::FETCH_ASSOC); return $rs['max']; } } ?>