#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
人民日报电子版交互式抓取 / 下载脚本

用法:
    python3 rmrb_opinion_scraper.py

功能:
    1. 抓取: 按日期浏览版面与文章标题，选择单篇/多篇下载为 Markdown
    2. 下载: 按日期选择版面，批量下载该版面全部文章
"""

from __future__ import annotations

import os
import re
import sys
import html
from datetime import datetime, date
from typing import List, Dict, Optional, Tuple
from urllib.parse import urljoin, urlparse
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
from html.parser import HTMLParser


# ---------------------------------------------------------------------------
# 常量
# ---------------------------------------------------------------------------

USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/120.0.0.0 Safari/537.36"
)
BASE_HOST = "https://paper.people.com.cn"
REQUEST_TIMEOUT = 25

# 旧版 / 新版 URL 模板
# 新版 PC: https://paper.people.com.cn/rmrb/pc/layout/YYYYMM/DD/node_01.html
# 旧版:    https://paper.people.com.cn/rmrb/html/YYYY-MM/DD/nbs.D110000renmrb_01.htm


# ---------------------------------------------------------------------------
# HTTP
# ---------------------------------------------------------------------------

def fetch(url: str) -> str:
    """请求页面，返回解码后的文本。失败时抛出异常。"""
    req = Request(url, headers={
        "User-Agent": USER_AGENT,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        "Referer": BASE_HOST + "/",
    })
    with urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
        raw = resp.read()
        # 优先从 Content-Type / meta 判断编码
        charset = None
        ctype = resp.headers.get("Content-Type", "")
        m = re.search(r"charset=([\w-]+)", ctype, re.I)
        if m:
            charset = m.group(1).strip()
        if not charset:
            head = raw[:2000].decode("ascii", errors="ignore")
            m = re.search(r'charset=["\']?([\w-]+)', head, re.I)
            if m:
                charset = m.group(1).strip()
        for enc in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
            if not enc:
                continue
            try:
                return raw.decode(enc)
            except (UnicodeDecodeError, LookupError):
                continue
        return raw.decode("utf-8", errors="replace")


def try_fetch(url: str) -> Optional[str]:
    try:
        return fetch(url)
    except (URLError, HTTPError, TimeoutError, OSError) as e:
        return None


# ---------------------------------------------------------------------------
# 简易 HTML 工具（无第三方依赖）
# ---------------------------------------------------------------------------

def strip_tags(text: str) -> str:
    text = re.sub(r"(?is)<script[^>]*>.*?</script>", "", text)
    text = re.sub(r"(?is)<style[^>]*>.*?</style>", "", text)
    text = re.sub(r"(?is)<!--.*?-->", "", text)
    text = re.sub(r"(?is)<br\s*/?>", "\n", text)
    text = re.sub(r"(?is)</p>", "\n\n", text)
    text = re.sub(r"(?is)</div>", "\n", text)
    text = re.sub(r"(?is)</h[1-6]>", "\n\n", text)
    text = re.sub(r"(?is)<[^>]+>", "", text)
    text = html.unescape(text)
    # 整理空白
    text = text.replace("\r\n", "\n").replace("\r", "\n")
    text = re.sub(r"[ \t]+\n", "\n", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def sanitize_filename(name: str) -> str:
    """生成合法文件名。"""
    name = name.strip()
    name = re.sub(r'[\\/:*?"<>|\r\n\t]', "_", name)
    name = re.sub(r"\s+", " ", name).strip(" .")
    if not name:
        name = "未命名文章"
    # 限制长度，避免路径过长
    if len(name) > 80:
        name = name[:80].rstrip()
    return name


# ---------------------------------------------------------------------------
# 日期与 URL
# ---------------------------------------------------------------------------

def parse_date_input(s: str) -> date:
    s = (s or "").strip()
    if not s:
        return date.today()
    for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"):
        try:
            return datetime.strptime(s, fmt).date()
        except ValueError:
            continue
    raise ValueError(f"无法解析日期: {s}，请使用 YYYY-MM-DD 格式")


def layout_urls_for_date(d: date) -> List[str]:
    """返回可能的首页版面 URL 列表（按优先级）。"""
    ymd = d.strftime("%Y%m")
    day = d.strftime("%d")
    y_m = d.strftime("%Y-%m")
    return [
        f"{BASE_HOST}/rmrb/pc/layout/{ymd}/{day}/node_01.html",
        f"{BASE_HOST}/rmrb/html/{y_m}/{day}/nbs.D110000renmrb_01.htm",
        f"http://paper.people.com.cn/rmrb/pc/layout/{ymd}/{day}/node_01.html",
        f"http://paper.people.com.cn/rmrb/html/{y_m}/{day}/nbs.D110000renmrb_01.htm",
    ]


def abs_url(base: str, href: str) -> str:
    if not href:
        return ""
    href = href.strip()
    if href.startswith("javascript:") or href.startswith("#"):
        return ""
    return urljoin(base, href)


# ---------------------------------------------------------------------------
# 解析版面列表
# ---------------------------------------------------------------------------

def parse_sections(page_html: str, page_url: str) -> List[Dict[str, str]]:
    """
    从版面页解析所有版面（板块）。
    返回: [{"name": "第01版：要闻", "url": "..."}, ...]
    """
    sections: List[Dict[str, str]] = []
    seen = set()

    # 模式1: 新版 swiper / right_title / pageLink
    # <a href="...node_XX.html" id="pageLink">第01版：要闻</a>
    patterns = [
        # 带 id=pageLink
        r'<a[^>]+href=["\']([^"\']*node_\d+\.html)["\'][^>]*>(.*?)</a>',
        r'<a[^>]+href=["\']([^"\']*nbs\.D110000renmrb_\d+\.htm)["\'][^>]*>(.*?)</a>',
        # 旧版
        r'<a[^>]+href=["\']([^"\']*nbs\.D110000renmrb[^"\']+)["\'][^>]*>(.*?)</a>',
    ]

    for pat in patterns:
        for m in re.finditer(pat, page_html, re.I | re.S):
            href = m.group(1)
            name = strip_tags(m.group(2))
            name = re.sub(r"\s+", " ", name).strip()
            url = abs_url(page_url, href)
            if not url or not name:
                continue
            # 过滤非版面链接
            if "node_" not in url and "nbs.D110000" not in url:
                continue
            key = url
            if key in seen:
                continue
            seen.add(key)
            sections.append({"name": name, "url": url})

    # 若未解析到，尝试从左侧目录 / 底部导航再扫一遍更宽松规则
    if not sections:
        for m in re.finditer(
            r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>\s*(第?\s*\d+\s*版[^<]*)\s*</a>',
            page_html,
            re.I | re.S,
        ):
            href, name = m.group(1), strip_tags(m.group(2))
            url = abs_url(page_url, href)
            if not url or url in seen:
                continue
            seen.add(url)
            sections.append({"name": name.strip(), "url": url})

    # 去重后若仍为空，把当前页当作唯一版面
    if not sections:
        # 尝试从标题提取版面名
        title_m = re.search(r"<title>(.*?)</title>", page_html, re.I | re.S)
        name = strip_tags(title_m.group(1)) if title_m else "第01版"
        sections.append({"name": name, "url": page_url})

    return sections


# ---------------------------------------------------------------------------
# 解析文章列表
# ---------------------------------------------------------------------------

def parse_articles(page_html: str, page_url: str) -> List[Dict[str, str]]:
    """
    从版面页解析文章列表。
    返回: [{"title": "...", "url": "..."}, ...]
    """
    articles: List[Dict[str, str]] = []
    seen = set()

    # 新版 content_ 链接 / 旧版 nw.D110000 链接
    link_patterns = [
        r'<a[^>]+href=["\']([^"\']*content_\d+\.html)["\'][^>]*>(.*?)</a>',
        r'<a[^>]+href=["\']([^"\']*nw\.D110000renmrb[^"\']+\.htm)["\'][^>]*>(.*?)</a>',
        r'<a[^>]+href=["\']([^"\']*content[^"\']*\.html?)["\'][^>]*>(.*?)</a>',
    ]

    for pat in link_patterns:
        for m in re.finditer(pat, page_html, re.I | re.S):
            href = m.group(1)
            title = strip_tags(m.group(2))
            title = re.sub(r"\s+", " ", title).strip()
            url = abs_url(page_url, href)
            if not url or not title:
                continue
            # 过滤导航类短标题
            if len(title) < 2:
                continue
            if title in ("上一版", "下一版", "返回目录", "放大", "缩小", "下载"):
                continue
            if url in seen:
                continue
            # 排除版面节点链接
            if re.search(r"node_\d+\.html", url) or "nbs.D110000" in url:
                continue
            seen.add(url)
            articles.append({"title": title, "url": url})

    # 从 map area 热点坐标解析（旧版报纸常见）
    if not articles:
        for m in re.finditer(
            r'<area[^>]+href=["\']([^"\']+)["\'][^>]*(?:alt|title)=["\']([^"\']+)["\']',
            page_html,
            re.I,
        ):
            href, title = m.group(1), html.unescape(m.group(2)).strip()
            url = abs_url(page_url, href)
            if not url or not title or url in seen:
                continue
            seen.add(url)
            articles.append({"title": title, "url": url})
        for m in re.finditer(
            r'<area[^>]*(?:alt|title)=["\']([^"\']+)["\'][^>]+href=["\']([^"\']+)["\']',
            page_html,
            re.I,
        ):
            title, href = html.unescape(m.group(1)).strip(), m.group(2)
            url = abs_url(page_url, href)
            if not url or not title or url in seen:
                continue
            seen.add(url)
            articles.append({"title": title, "url": url})

    return articles


# ---------------------------------------------------------------------------
# 解析文章正文
# ---------------------------------------------------------------------------

def parse_article_content(page_html: str, page_url: str) -> Tuple[str, str, str]:
    """
    返回 (title, content_text, source_info)
    """
    # 标题
    title = ""
    for pat in (
        r'<h1[^>]*class=["\'][^"\']*article[^"\']*["\'][^>]*>(.*?)</h1>',
        r'<h1[^>]*>(.*?)</h1>',
        r'<div[^>]+class=["\'][^"\']*title["\'][^>]*>(.*?)</div>',
        r"<title>(.*?)</title>",
    ):
        m = re.search(pat, page_html, re.I | re.S)
        if m:
            title = strip_tags(m.group(1))
            title = re.sub(r"\s*[-_|].*$", "", title).strip()  # 去掉站点后缀
            if title:
                break

    # 正文候选区域
    content = ""
    content_patterns = [
        r'<div[^>]+id=["\']ozoom["\'][^>]*>(.*?)</div>\s*</div>',
        r'<div[^>]+class=["\'][^"\']*article[_-]?content[^"\']*["\'][^>]*>(.*?)</div>',
        r'<div[^>]+id=["\']articleContent["\'][^>]*>(.*?)</div>',
        r'<div[^>]+class=["\'][^"\']*content["\'][^>]*>(.*?)</div>',
        r'<div[^>]+class=["\'][^"\']*text[_-]?content[^"\']*["\'][^>]*>(.*?)</div>',
        r'<founder-content[^>]*>(.*?)</founder-content>',
    ]
    for pat in content_patterns:
        m = re.search(pat, page_html, re.I | re.S)
        if m:
            block = m.group(1)
            # 去掉内嵌脚本样式
            text = strip_tags(block)
            if len(text) > 30:
                content = text
                break

    # 兜底：取所有 <p>
    if not content:
        ps = re.findall(r"<p[^>]*>(.*?)</p>", page_html, re.I | re.S)
        paras = [strip_tags(p) for p in ps]
        paras = [p for p in paras if p and len(p) > 1]
        # 过滤页脚噪声
        noise = ("人民网", "返回目录", "上一版", "下一版", "放大", "缩小")
        paras = [p for p in paras if not any(n == p for n in noise)]
        content = "\n\n".join(paras)

    # 来源 / 日期信息
    source = ""
    for pat in (
        r'<div[^>]+class=["\'][^"\']*sec[^"\']*["\'][^>]*>(.*?)</div>',
        r'<div[^>]+class=["\'][^"\']*date["\'][^>]*>(.*?)</div>',
        r'<span[^>]+class=["\'][^"\']*date["\'][^>]*>(.*?)</span>',
    ):
        m = re.search(pat, page_html, re.I | re.S)
        if m:
            source = strip_tags(m.group(1))
            source = re.sub(r"\s+", " ", source).strip()
            if source:
                break

    if not title:
        title = "未命名文章"
    return title, content, source


def article_to_markdown(title: str, content: str, source: str, url: str, d: date) -> str:
    lines = [
        f"# {title}",
        "",
        f"- 日期: {d.isoformat()}",
        f"- 来源: 人民日报电子版" + (f" | {source}" if source else ""),
        f"- 链接: {url}",
        "",
        "---",
        "",
        content if content else "_（未能解析到正文，请打开链接查看）_",
        "",
    ]
    return "\n".join(lines)


# ---------------------------------------------------------------------------
# 数据层：加载某日全部版面
# ---------------------------------------------------------------------------

class DayPaper:
    def __init__(self, d: date):
        self.date = d
        self.home_url: Optional[str] = None
        self.sections: List[Dict[str, str]] = []  # name, url
        self._section_html: Dict[str, str] = {}   # url -> html
        self._section_articles: Dict[str, List[Dict[str, str]]] = {}

    def load(self) -> None:
        last_err = None
        html_text = None
        for url in layout_urls_for_date(self.date):
            try:
                html_text = fetch(url)
                self.home_url = url
                break
            except Exception as e:
                last_err = e
                html_text = None
        if not html_text or not self.home_url:
            raise RuntimeError(
                f"无法获取 {self.date.isoformat()} 的人民日报电子版。"
                f" 请确认日期是否有报纸（周末/节假日版面较少），网络是否可用。"
                f" 详情: {last_err}"
            )
        self.sections = parse_sections(html_text, self.home_url)
        # 缓存首页 html
        self._section_html[self.home_url] = html_text

    def get_section_html(self, section_url: str) -> str:
        if section_url in self._section_html:
            return self._section_html[section_url]
        text = fetch(section_url)
        self._section_html[section_url] = text
        return text

    def get_articles(self, section: Dict[str, str]) -> List[Dict[str, str]]:
        url = section["url"]
        if url in self._section_articles:
            return self._section_articles[url]
        page = self.get_section_html(url)
        arts = parse_articles(page, url)
        self._section_articles[url] = arts
        return arts


# ---------------------------------------------------------------------------
# 交互工具
# ---------------------------------------------------------------------------

def prompt(msg: str, default: str = "") -> str:
    try:
        if default:
            s = input(f"{msg} [{default}]: ").strip()
            return s if s else default
        return input(f"{msg}: ").strip()
    except (EOFError, KeyboardInterrupt):
        print("\n已取消。")
        sys.exit(0)


def prompt_date(msg: str = "请输入日期 (YYYY-MM-DD，回车默认当天)") -> date:
    while True:
        s = prompt(msg, date.today().isoformat())
        try:
            return parse_date_input(s)
        except ValueError as e:
            print(f"  ! {e}")


def parse_selection(s: str, max_n: int) -> List[int]:
    """
    解析用户选择: "1" / "1-3" / "1,3,5" / "1,3-5"
    返回 1-based 索引列表（已去重、排序）。
    """
    s = (s or "").strip()
    if not s:
        return []
    chosen = set()
    for part in re.split(r"[，,]\s*", s):
        part = part.strip()
        if not part:
            continue
        if re.fullmatch(r"\d+", part):
            n = int(part)
            if 1 <= n <= max_n:
                chosen.add(n)
            else:
                raise ValueError(f"序号超出范围: {n} (1-{max_n})")
        elif re.fullmatch(r"\d+\s*[-~～到至]\s*\d+", part):
            a, b = re.split(r"[-~～到至]", part)
            a, b = int(a.strip()), int(b.strip())
            if a > b:
                a, b = b, a
            for n in range(a, b + 1):
                if 1 <= n <= max_n:
                    chosen.add(n)
                else:
                    raise ValueError(f"序号超出范围: {n} (1-{max_n})")
        else:
            raise ValueError(f"无法解析选择: {part}")
    return sorted(chosen)


def print_numbered(items: List[str], title: str = "") -> None:
    if title:
        print(f"\n{title}")
        print("-" * 40)
    for i, name in enumerate(items, 1):
        print(f"  {i}. {name}")
    print()


def download_article(art: Dict[str, str], d: date, save_dir: str) -> Dict:
    """下载单篇文章，返回详情 dict。"""
    info = {
        "title": art.get("title", ""),
        "url": art["url"],
        "ok": False,
        "path": "",
        "error": "",
    }
    try:
        page = fetch(art["url"])
        title, content, source = parse_article_content(page, art["url"])
        if not title or title == "未命名文章":
            title = art.get("title") or title
        info["title"] = title
        md = article_to_markdown(title, content, source, art["url"], d)
        fname = sanitize_filename(title) + ".md"
        path = os.path.join(save_dir, fname)
        # 重名则追加序号
        if os.path.exists(path):
            base = sanitize_filename(title)
            k = 2
            while os.path.exists(os.path.join(save_dir, f"{base}_{k}.md")):
                k += 1
            path = os.path.join(save_dir, f"{base}_{k}.md")
        with open(path, "w", encoding="utf-8") as f:
            f.write(md)
        info["ok"] = True
        info["path"] = path
    except Exception as e:
        info["error"] = str(e)
    return info


def print_download_report(results: List[Dict]) -> None:
    ok = [r for r in results if r["ok"]]
    fail = [r for r in results if not r["ok"]]
    print("\n" + "=" * 50)
    print("下载完成详情")
    print("=" * 50)
    print(f"成功: {len(ok)} 篇  |  失败: {len(fail)} 篇  |  合计: {len(results)} 篇")
    if ok:
        print("\n[成功]")
        for r in ok:
            print(f"  ✓ {r['title']}")
            print(f"    -> {r['path']}")
    if fail:
        print("\n[失败]")
        for r in fail:
            print(f"  ✗ {r.get('title') or r['url']}")
            print(f"    原因: {r['error']}")
    print("=" * 50 + "\n")


# ---------------------------------------------------------------------------
# 模式 1: 抓取
# ---------------------------------------------------------------------------

def mode_scrape(save_dir: str) -> None:
    print("\n>>> 模式: 1. 抓取（浏览后选择文章下载）")
    d = prompt_date()
    print(f"\n正在获取 {d.isoformat()} 的版面信息...")
    paper = DayPaper(d)
    try:
        paper.load()
    except RuntimeError as e:
        print(f"错误: {e}")
        return

    if not paper.sections:
        print("未解析到任何版面。")
        return

    print_numbered([s["name"] for s in paper.sections], f"{d.isoformat()} 版面列表")

    while True:
        raw = prompt("请选择要查看的版面序号 (q 退出)")
        if raw.lower() in ("q", "quit", "exit"):
            return
        try:
            idxs = parse_selection(raw, len(paper.sections))
            if len(idxs) != 1:
                print("  请输入单个版面序号。")
                continue
            section = paper.sections[idxs[0] - 1]
            break
        except ValueError as e:
            print(f"  ! {e}")

    print(f"\n正在获取版面「{section['name']}」的文章列表...")
    try:
        articles = paper.get_articles(section)
    except Exception as e:
        print(f"错误: 获取文章列表失败: {e}")
        return

    if not articles:
        print("该版面下未解析到文章。")
        return

    print_numbered(
        [a["title"] for a in articles],
        f"「{section['name']}」文章列表",
    )
    print("选择说明: 单篇如 2；多篇如 1-3 或 1,3,5")

    while True:
        raw = prompt("请选择要下载的文章序号 (q 退出)")
        if raw.lower() in ("q", "quit", "exit"):
            return
        try:
            idxs = parse_selection(raw, len(articles))
            if not idxs:
                print("  未选择任何文章。")
                continue
            break
        except ValueError as e:
            print(f"  ! {e}")

    selected = [articles[i - 1] for i in idxs]
    print(f"\n开始下载 {len(selected)} 篇文章到: {save_dir}")
    results = []
    for i, art in enumerate(selected, 1):
        print(f"  [{i}/{len(selected)}] {art['title']} ...", end=" ", flush=True)
        r = download_article(art, d, save_dir)
        results.append(r)
        print("OK" if r["ok"] else f"失败: {r['error']}")
    print_download_report(results)


# ---------------------------------------------------------------------------
# 模式 2: 下载
# ---------------------------------------------------------------------------

def mode_download(save_dir: str) -> None:
    print("\n>>> 模式: 2. 下载（按版面批量下载全部文章）")
    print("提示: 若只想下载单篇文章，请输入「back」切换到抓取模式。")
    d = prompt_date()
    print(f"\n正在获取 {d.isoformat()} 的版面信息...")
    paper = DayPaper(d)
    try:
        paper.load()
    except RuntimeError as e:
        print(f"错误: {e}")
        return

    if not paper.sections:
        print("未解析到任何版面。")
        return

    print_numbered([s["name"] for s in paper.sections], f"{d.isoformat()} 版面列表")
    print("选择说明: 单版如 2；多版如 1-3 或 1,3,5")
    print("提示: 若只想下载单篇文章，请输入「back」切换到抓取模式。")

    def wants_scrape_mode(text: str) -> bool:
        t = text.strip().lower().replace(" ", "")
        return t in ("back", "b", "抓取", "scrape")

    while True:
        raw = prompt("请选择要下载全部文章的版面序号 (q 退出 / back 抓取)")
        if raw.lower() in ("q", "quit", "exit"):
            return
        if wants_scrape_mode(raw):
            mode_scrape(save_dir)
            return
        try:
            idxs = parse_selection(raw, len(paper.sections))
            if not idxs:
                print("  未选择任何版面。")
                continue
            break
        except ValueError as e:
            print(f"  ! {e}")

    chosen_sections = [paper.sections[i - 1] for i in idxs]
    all_results: List[Dict] = []

    for sec in chosen_sections:
        print(f"\n>>> 版面: {sec['name']}")
        try:
            articles = paper.get_articles(sec)
        except Exception as e:
            print(f"  获取文章列表失败: {e}")
            continue
        if not articles:
            print("  该版面无文章。")
            continue
        print(f"  共 {len(articles)} 篇，开始下载...")
        for i, art in enumerate(articles, 1):
            print(f"  [{i}/{len(articles)}] {art['title']} ...", end=" ", flush=True)
            r = download_article(art, d, save_dir)
            all_results.append(r)
            print("OK" if r["ok"] else f"失败: {r['error']}")

    if all_results:
        print_download_report(all_results)
    else:
        print("\n没有下载任何文章。")


# ---------------------------------------------------------------------------
# 主入口
# ---------------------------------------------------------------------------

def main() -> None:
    # Windows 控制台尽量使用 utf-8
    try:
        sys.stdout.reconfigure(encoding="utf-8")
        sys.stdin.reconfigure(encoding="utf-8")
    except Exception:
        pass

    save_dir = os.getcwd()
    print("=" * 50)
    print("  人民日报电子版 · 交互式抓取/下载工具")
    print("=" * 50)
    print(f"默认保存目录: {save_dir}")
    print()
    print("请选择模式:")
    print("  1. 抓取  - 浏览版面与标题，按需下载文章")
    print("  2. 下载  - 按版面批量下载全部文章")
    print()

    while True:
        choice = prompt("请输入 1 或 2 (q 退出)")
        if choice.lower() in ("q", "quit", "exit"):
            print("再见。")
            return
        if choice in ("1", "1.", "抓取"):
            mode_scrape(save_dir)
            break
        if choice in ("2", "2.", "下载"):
            mode_download(save_dir)
            break
        print("  请输入 1 或 2。")

    # 结束后询问是否继续
    while True:
        again = prompt("是否继续? (1 抓取 / 2 下载 / q 退出)", "q")
        if again.lower() in ("q", "quit", "exit", ""):
            print("再见。")
            return
        if again in ("1", "抓取"):
            mode_scrape(save_dir)
        elif again in ("2", "下载"):
            mode_download(save_dir)
        else:
            print("  请输入 1 / 2 / q。")


if __name__ == "__main__":
    main()