"""经本机浏览器取页(ADR-0065):对着假的 CDP 服务验证握手、帧、验证页三态、年龄门与进程生命周期。""" import base64 import hashlib import json import socket import struct import tempfile import threading import unittest from pathlib import Path from unittest.mock import patch from peach import browser_transport from peach.browser_transport import (BrowserTransport, BrowserUnavailable, ChallengeUnsolved, PageTimeout, _WebSocket, find_browser, proxy_flags) from peach.http import HttpRequest PAGE_HTML = "FC2-PPV-1 | FC2PPV Databaseok" def document_reply(url: str, html: str = PAGE_HTML, status: int = 200, kind: str = "text/html") -> str: return json.dumps({"url": url, "type": kind, "status": status, "html": html}) def state_reply(title: str, ready: str = "complete", head: str = "", url: str = "https://x/") -> str: return json.dumps({"title": title, "ready": ready, "head": head, "url": url}) def is_state(params) -> bool: return "document.readyState" in params["expression"] def is_document(params) -> bool: return "getEntriesByType" in params["expression"] class FakeCdp(threading.Thread): """一个端口同时充当 `/json` 目标列表和页面的 WebSocket 端:回包按 `handler(method, params)`。""" def __init__(self, handler): super().__init__(daemon=True) self.handler = handler self.calls: list[tuple[str, dict]] = [] self.listener = socket.socket() self.listener.bind(("127.0.0.1", 0)) self.listener.listen(5) self.port = self.listener.getsockname()[1] self.start() def targets(self, port: int) -> list[dict]: assert port == self.port return [{"type": "page", "id": "T1", "webSocketDebuggerUrl": f"ws://127.0.0.1:{self.port}/devtools/page/T1"}] def run(self): while True: try: connection, _ = self.listener.accept() except OSError: return threading.Thread(target=self.serve, args=(connection,), daemon=True).start() def serve(self, connection: socket.socket): head = b"" while b"\r\n\r\n" not in head: chunk = connection.recv(4096) if not chunk: return head += chunk request, rest = head.split(b"\r\n\r\n", 1) key = next(line.split(b":", 1)[1].strip() for line in request.split(b"\r\n") if line.lower().startswith(b"sec-websocket-key")) accept = base64.b64encode(hashlib.sha1(key + b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()) connection.sendall(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" b"Sec-WebSocket-Accept: " + accept + b"\r\n\r\n") buffer = rest while True: frame, buffer = self.read_frame(connection, buffer) if frame is None: connection.close() return message = json.loads(frame) self.calls.append((message["method"], message.get("params", {}))) result = self.handler(message["method"], message.get("params", {})) reply = json.dumps({"id": message["id"], "result": result}).encode() length = len(reply) if length < 126: header = bytes([0x81, length]) elif length < 65536: header = bytes([0x81, 126]) + struct.pack(">H", length) else: header = bytes([0x81, 127]) + struct.pack(">Q", length) connection.sendall(header + reply) @staticmethod def read_frame(connection, buffer): def need(size): nonlocal buffer while len(buffer) < size: chunk = connection.recv(65536) if not chunk: raise ConnectionError buffer += chunk out, buffer = buffer[:size], buffer[size:] return out try: first, second = need(2) opcode, size = first & 0x0F, second & 0x7F if size == 126: size = struct.unpack(">H", need(2))[0] elif size == 127: size = struct.unpack(">Q", need(8))[0] mask = need(4) if second & 0x80 else b"\0\0\0\0" data = bytes(byte ^ mask[index % 4] for index, byte in enumerate(need(size))) except ConnectionError: return None, buffer if opcode == 0x8: return None, buffer return data.decode(), buffer def close(self): self.listener.close() class FakeProcess: def __init__(self, port_file: Path, port: int): self.port_file, self.port, self.returncode = port_file, port, None self.killed = False def poll(self): return self.returncode def wait(self, timeout=None): self.returncode = 0 if self.returncode is None else self.returncode return self.returncode def kill(self): self.killed = True self.returncode = -9 class Clock: def __init__(self): self.now = 1000.0 def __call__(self): return self.now def sleep(self, seconds): self.now += seconds class BrowserTransportTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) self.profile = Path(self.temporary.name) / "browser" self.launches: list[list[str]] = [] self.processes: list[FakeProcess] = [] self.clock = Clock() def make(self, handler, executable="C:/fake/chrome.exe", **options) -> tuple[BrowserTransport, FakeCdp]: server = FakeCdp(handler) self.addCleanup(server.close) def popen(command, **_kwargs): self.launches.append(command) port_file = self.profile / "DevToolsActivePort" port_file.write_text(f"{server.port}\n/devtools/browser/abc\n", encoding="utf-8") process = FakeProcess(port_file, server.port) self.processes.append(process) return process transport = BrowserTransport(executable, self.profile, ("--no-proxy-server",), popen=popen, targets=server.targets, clock=self.clock, sleep=self.clock.sleep, auto_seconds=40, click_seconds=120, idle_seconds=600, **options) self.addCleanup(transport.close) return transport, server @staticmethod def generic(method, params): if method == "Browser.getWindowForTarget": return {"windowId": 7} return {} def test_the_browser_navigates_to_the_address_and_the_document_comes_back_as_a_response(self): """取页就是让浏览器导航过去,DOM 解析完把最终地址、状态码与 HTML 读回来;Cookie、UA 由浏览器自己带。""" final = "https://javten.com/tw/video/1/id2/x" def handler(method, params): if method == "Runtime.evaluate" and is_document(params): return {"result": {"value": document_reply(final, PAGE_HTML, 200, "text/html")}} if method == "Runtime.evaluate": return {"result": {"value": state_reply("JAVten", "interactive", url=final)}} return self.generic(method, params) transport, server = self.make(handler) response = transport(HttpRequest("GET", "https://javten.com/search?kw=2", {"Cookie": "secret=1", "User-Agent": "x", "Accept-Language": "ja,en;q=0.9", "Referer": "https://javten.com/"}), 12.5, 4096) self.assertEqual((response.status, response.body.decode(), response.url), (200, PAGE_HTML, final)) self.assertEqual(response.headers["content-type"], "text/html; charset=utf-8") self.assertNotIn("secret", json.dumps(server.calls), "Cookie 与 UA 不进浏览器,由它自己填") extra = [params["headers"] for method, params in server.calls if method == "Network.setExtraHTTPHeaders"] self.assertEqual(extra, [{"Accept-Language": "ja,en;q=0.9"}], "会话上只挂语言头:JAVten 按 Accept-Language 决定回日文原页还是译文页") self.assertEqual([params for method, params in server.calls if method == "Page.navigate"], [{"url": "https://javten.com/search?kw=2", "referrer": "https://javten.com/"}], "上一跳只作主文档的 Referer:挂在会话上会让 Turnstile 勾选框的 iframe 被浏览器拦下") # 启动参数:调试口、关掉自动化标记与它的警告条、不同步、profile 在凭据根下、窗口在屏幕外、来源的连接方式。 command = self.launches[0] for flag in ("--remote-debugging-port=0", "--disable-blink-features=AutomationControlled", "--test-type", "--disable-sync", f"--user-data-dir={self.profile}", "--window-position=-32000,-32000", "--no-proxy-server"): self.assertIn(flag, command) self.assertNotIn("--inprivate", command, "Chrome 不会拿系统账号登录,用持久 profile 留住 cf_clearance") self.assertEqual([call[0] for call in server.calls], ["Page.enable", "Network.enable", "Network.setBlockedURLs", "Browser.getWindowForTarget", "Network.setExtraHTTPHeaders", "Page.navigate", "Runtime.evaluate", "Runtime.evaluate"]) blocked = server.calls[2][1]["urls"] self.assertTrue({"*.jpg", "*.webp", "*.woff2", "*.mp4"} <= set(blocked), "图片、字体与媒体不下载") transport(HttpRequest("GET", "https://fc2ppv-db.com/ja/videos/1", {}), 10, 4) self.assertEqual([params for method, params in server.calls if method == "Page.navigate"][1:], [{"url": "https://fc2ppv-db.com/ja/videos/1"}], "每条请求各导航一次;没有上一跳就不传 referrer") with self.assertRaises(BrowserUnavailable): transport(HttpRequest("POST", "https://javten.com/", {}), 10, 4096) def test_an_attended_window_opens_on_screen_keeps_its_profile_and_loads_everything(self): """给人登录用的窗口(ADR-0093):开在屏幕内、Edge 也不用 InPrivate,图片与字体照常加载。""" server = FakeCdp(self.generic) self.addCleanup(server.close) def popen(command, **_kwargs): self.launches.append(command) (self.profile / "DevToolsActivePort").write_text(f"{server.port}\n/devtools/browser/abc\n", encoding="utf-8") return FakeProcess(self.profile / "DevToolsActivePort", server.port) browser = browser_transport._Browser("C:/fake/msedge.exe", self.profile, (), popen=popen, targets=server.targets, sleep=self.clock.sleep, attended=True) browser.start() self.addCleanup(browser.close) command = self.launches[0] self.assertIn(f"--window-position={browser_transport.ONSCREEN[0]},{browser_transport.ONSCREEN[1]}", command) self.assertIn(f"--user-data-dir={self.profile}", command) self.assertNotIn("--inprivate", command, "登录要的 localStorage 必须落在 profile 里") self.assertNotIn("Network.setBlockedURLs", [method for method, _params in server.calls], "登录页和人机验证靠图片与字体渲染") def test_the_body_is_cut_at_the_limit_and_a_missing_status_reads_as_ok(self): def handler(method, params): if method == "Runtime.evaluate" and is_document(params): return {"result": {"value": json.dumps({"url": "https://javten.com/", "type": "text/html", "status": 0, "html": "字" * 100})}} if method == "Runtime.evaluate": return {"result": {"value": state_reply("JAVten")}} return self.generic(method, params) transport, _server = self.make(handler) response = transport(HttpRequest("GET", "https://javten.com/", {}), 10, 30) self.assertEqual((response.status, len(response.body)), (200, 31), "多读一个字节让调用方判断超限") def test_a_challenge_page_is_waited_out_without_touching_the_window(self): """导航落到验证页:每秒看一次,浏览器自己过了就读文档;自动过的验证不动窗口、不提醒。""" states = iter([state_reply("Just a moment...", "interactive"), state_reply("请稍候…"), state_reply("FC2-PPV-1 | FC2PPV Database", url="https://fc2ppv-db.com/ja/videos/1")]) def handler(method, params): if method == "Runtime.evaluate" and is_document(params): return {"result": {"value": document_reply("https://fc2ppv-db.com/ja/videos/1")}} if method == "Runtime.evaluate": return {"result": {"value": next(states)}} return self.generic(method, params) transport, server = self.make(handler) response = transport(HttpRequest("GET", "https://fc2ppv-db.com/ja/videos/1", {}), 10, 4096) self.assertEqual((response.status, response.body.decode()), (200, PAGE_HTML)) self.assertEqual(len([1 for method, _ in server.calls if method == "Page.navigate"]), 1) self.assertFalse(any(method == "Browser.setWindowBounds" for method, _ in server.calls)) self.assertFalse(any(method == "Network.deleteCookies" for method, _ in server.calls), "几秒就过的验证不动 cookie") self.assertEqual(browser_transport.attention(), []) def test_a_stuck_challenge_gets_the_sites_cookies_cleared_and_reloaded_before_anyone_is_asked(self): """验证页转了 15 秒还没过:只删这一站的 cookie、重新导航,重载后过了就照常读文档,窗口不动、不提醒。""" cleared_at = [] def handler(method, params): if method == "Network.getCookies": return {"cookies": [{"name": "cf_clearance", "domain": "javten.com", "path": "/"}, {"name": "cf_chl_rc_ni", "domain": "javten.com", "path": "/"}]} if method == "Network.deleteCookies": cleared_at.append(self.clock.now - started) if method == "Runtime.evaluate" and is_document(params): return {"result": {"value": document_reply("https://javten.com/search?kw=1")}} if method == "Runtime.evaluate": if cleared_at and self.clock.now - started >= cleared_at[-1] + 5: return {"result": {"value": state_reply("JAVten", url="https://javten.com/search?kw=1")}} return {"result": {"value": state_reply("しばらくお待ちください...", head="