"""站点追更连接器的隔离测试。
夹具的字段名与 DOM 结构取自 2026-08-25 对真实站点的实测响应(kemono.cr
`fanbox/30917150`、rule34video.com `/models/lazyprocrastinator/`、f95zone
thread 50685),不是凭记忆构造的形状。测试本身不联网:transport 全部注入。
"""
import json
import hashlib
import os
import tempfile
import unittest
import urllib.parse
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import patch
from peach import follow_sources
from peach.follow import FollowHistoryEnd, FollowSourceError, FollowSourceRateLimited
from peach.follow_secrets import Credential, CredentialError, CredentialStore
from peach.follow_sources import (
USER_AGENT, F95ZoneConnector, FanboxConnector, IwaraConnector, KemonoConnector,
PatreonConnector, Rule34PahealConnector, Rule34VideoConnector, Rule34XxxConnector,
SimpCityConnector, SubscribeStarConnector, build_connector,
_iso_from_relative, origin_group_key, parse_source_url,
)
from peach.http import HttpResponse
from support.mp4 import minimal_mp4
def _transport(status=200, headers=None, body=b"", record=None):
def call(request, timeout, max_bytes):
if record is not None:
record.append(request)
return HttpResponse(status, headers or {}, body)
return call
KEMONO_POSTS = json.dumps([
{"id": "11406814", "user": "30917150", "service": "fanbox",
"title": "Villainous Valentine's Day 7", "substring": "",
"published": "2026-02-14T21:51:10", "edited": None,
"file": {"name": "a.png", "path": "/1c/fa/1cfae7.png"},
"attachments": [{"name": "a.zip", "path": "/1c/fa/1cfae7.zip"}]},
# 第二条只有附件、没有正文文件,`media_url` 要能从附件里取到。连附件也没有的帖子
# 会被判为「不是 release」丢掉,拿那种形状当固定件会让别的断言都少一条。
{"id": "11400490", "user": "30917150", "service": "fanbox",
"title": "Villainous Valentine's Day 6", "published": "2026-02-13T21:30:28",
"file": {}, "attachments": [{"name": "b.png", "path": "/2f/46/2f468d.png"}]},
]).encode()
RULE34VIDEO_HTML = b"""
4 weeks ago
"""
RULE34VIDEO_DETAIL_HTML = b"""
3D
Final Fantasy
LazyProcrastinator
deep throat
breast squeeze
"""
# XML 形态取自 2026-11-03 对 api.rule34.xxx 的实测:只有它带 `created_at`,带时区偏移。
RULE34XXX_XML_FIRST_TWO = (
b''
b''
b''
b"")
RULE34XXX_XML = RULE34XXX_XML_FIRST_TWO.replace(b"", (
b''
b''))
# 搜索表单与结果页的结构取自 2026-09-01 对 f95zone.to 的实测:关键词 `Ria_neearts`
# 在 latest_data.php 的五个分类里全为空,站内搜索命中
# `/threads/ria-collection-2026-08-03-ria_neearts.146348/`。结果按帖子返回,
# 所以同一个线程会出现多条。
F95_SEARCH_FORM = b""""""
F95_SEARCH_RESULTS = b"""
-
Overview: Ria_neearts is a 3D artist from Italy.
-
"""
# 2026-09-12 实测 `63802` 的首楼:发帖人是搬运工,作者本人的地址写在正文的链接区
# 里,同一个人在 X 上两个账号并列。
F95_OPENING_POST = b"""Strauzek | F95zone
CollectionPinup
Strauzek Collection [2026-09-04] [Mr_Strauz]
"""
F95_HTML = b"""Collection - Video - Lazy | F95zone
CollectionVideo
Lazy Procrastinator Collection [2026-06-28] [LazyProcrastinator/LazyProcrast]
Jkhomie1198 said:
Gofile
Click to expand...
Genre spoilers first page have futa spoiler.
"""
FANBOX_JSON = json.dumps({"body": {"posts": [
{"id": "12489354", "title": "MobiusFF Sarah Animations", "feeRequired": 0,
"publishedDatetime": "2026-08-26T23:34:51+09:00", "isRestricted": False,
"user": {"name": "InitialA"}, "cover": {"url": "https://img.test/cover.jpg"},
"excerpt": "Public release"},
{"id": "12400000", "title": "Paid preview", "feeRequired": 500,
"publishedDatetime": "2026-08-20T00:00:00+09:00", "isRestricted": True},
]}}).encode()
#: 视频帖的正文只有一个 file 块,`fileMap` 里没有 `thumbnailUrl`——FANBOX 不给
#: 视频出图。列表接口那张 cover 是这篇唯一能显示的缩略图。
FANBOX_VIDEO_DETAIL_JSON = json.dumps({"body": {"post": {"type": "file", "body": {
"blocks": [{"type": "file", "fileId": "clip"}],
"fileMap": {"clip": {"id": "clip", "name": "scene", "extension": "mp4",
"url": "https://downloads.fanbox.cc/scene.mp4", "size": 9991}},
}}}}).encode()
FANBOX_DETAIL_JSON = json.dumps({"body": {"post": {"type": "article", "body": {
"blocks": [
{"type": "p", "text": "gofile - https://gofile.io/d/OS2Qz9"},
{"type": "image", "imageId": "one"},
{"type": "image", "imageId": "two"},
],
"imageMap": {
"one": {"originalUrl": "https://downloads.fanbox.cc/one.jpg",
"thumbnailUrl": "https://downloads.fanbox.cc/one-thumb.jpg",
"width": 1920, "height": 1080},
"two": {"originalUrl": "https://downloads.fanbox.cc/two.jpg",
"thumbnailUrl": "https://downloads.fanbox.cc/two-thumb.jpg"},
},
}}}}).encode()
GOFILE_JSON = json.dumps({"status": "ok", "data": {"type": "folder", "children": {
"v1": {"id": "v1", "type": "file", "name": "one.mp4", "mimetype": "video/mp4",
"link": "https://store1.gofile.io/download/one.mp4",
"thumbnail": "https://store1.gofile.io/one.jpg", "size": 123},
"v2": {"id": "v2", "type": "file", "name": "two.mp4", "mimetype": "video/mp4",
"link": "https://store1.gofile.io/download/two.mp4"},
"txt": {"id": "txt", "type": "file", "name": "readme.txt", "mimetype": "text/plain",
"link": "https://store1.gofile.io/download/readme.txt"},
}}}).encode()
SUBSCRIBESTAR_HTML = b"""
InitialA
MobiusFF Sarah Animations
Posted for FREE tiers
"""
PATREON_HTML = b""""""
# 属性名与 DOM 形状取自 2026-09-08 对 simpcity.cr 真实线程页的实测:`img.bbImage` 的
# `data-url` 是原图、`src` 是缩略图,分页在 `.pageNav-page a[href]`,表情是 `img.smilie`。
# 内容本身是编的。
SIMPCITY_FIRST_HTML = b"""
Sample Creator | SimpCity Forums
OnlyFans Sample Creator
First page post
"""
SIMPCITY_LAST_HTML = b"""
Sample Creator | SimpCity Forums
OnlyFans Sample Creator
uploader1 said:
Gofile Click to expand...
thanks!
"""
SIMPCITY_GUEST_HTML = b"""
Sample Creator
"""
SIMPCITY_SEARCH_FORM = b"""
"""
# 2026-09-08 实测 `solazola` 按标题命中的两行:资源线程与讨论帖各一条,标题前挂着版块标签。
SIMPCITY_SEARCH_RESULTS = b"""
HatEF Thread Sep 11, 2024 Replies: 124 Forum: Model Discussion
"""
class OfficialConnectorTests(unittest.TestCase):
def test_fanbox_keeps_only_public_free_posts(self):
seen = []
def route(request):
seen.append(request)
return HttpResponse(200, {}, FANBOX_DETAIL_JSON if "post.info" in request.url
else FANBOX_JSON)
result = FanboxConnector(transport=_routed(route)).fetch("ffxivinitiala")
self.assertIn("creatorId=ffxivinitiala", seen[0].url)
self.assertEqual(seen[0].headers["Origin"], "https://www.fanbox.cc")
self.assertIn("Mozilla/5.0", seen[0].headers["User-Agent"])
self.assertIn("Mozilla/5.0", seen[1].headers["User-Agent"])
self.assertEqual(len(result.candidates), 1)
self.assertEqual(result.skipped, 1)
self.assertEqual(result.candidates[0].group_hint, "fanbox:12489354")
self.assertEqual(result.candidates[0].published_at, "2026-08-26T14:34:51Z")
self.assertEqual(len(result.candidates[0].extra["media_items"]), 2)
self.assertEqual(result.candidates[0].extra["media_items"][0]["width"], 1920)
self.assertEqual(result.candidates[0].extra["media_items"][1]["width"], None)
self.assertEqual(result.candidates[0].extra["links"],
["https://gofile.io/d/OS2Qz9"])
self.assertEqual(result.candidates[0].extra["post_type"], "article")
self.assertTrue(result.candidates[0].extra["media_dims"])
self.assertEqual(result.candidates[0].extra["image_count"], 2)
self.assertEqual(result.candidates[0].extra["video_count"], 0)
self.assertEqual(result.candidates[0].extra["file_count"], 0)
self.assertEqual(result.probed, 1)
def test_fanbox_history_pages_follow_the_site_cursor_list(self):
seen = []
def route(request):
seen.append(request.url)
if "paginateCreator" in request.url:
return HttpResponse(200, {}, json.dumps({"body": {"pageUrls": [
"https://api.fanbox.cc/post.listCreator?creatorId=x&firstId=11",
"https://api.fanbox.cc/post.listCreator?creatorId=x&firstId=21",
]}}).encode())
return HttpResponse(200, {}, FANBOX_JSON)
result = FanboxConnector(transport=_routed(route)).fetch("ffxivinitiala", page=2)
self.assertIn("paginateCreator", seen[0])
self.assertIn("firstId=21", seen[1])
# 其余请求是第二阶段的详情补全。
self.assertTrue(all("post.info" in url for url in seen[2:]))
self.assertEqual(result.candidates[0].external_id, "12489354")
def test_fanbox_history_walk_ends_when_the_cursor_list_runs_out(self):
def route(request):
return HttpResponse(200, {}, json.dumps({"body": {"pageUrls": [
"https://api.fanbox.cc/post.listCreator?creatorId=x&firstId=11",
]}}).encode())
connector = FanboxConnector(transport=_routed(route))
with self.assertRaises(FollowHistoryEnd):
connector.fetch("ffxivinitiala", page=2)
def test_fanbox_history_walk_ends_on_an_empty_cursor_page(self):
def route(request):
if "paginateCreator" in request.url:
return HttpResponse(200, {}, json.dumps({"body": {"pageUrls": [
"https://api.fanbox.cc/post.listCreator?creatorId=x&firstId=11",
]}}).encode())
return HttpResponse(200, {}, json.dumps({"body": {"posts": []}}).encode())
connector = FanboxConnector(transport=_routed(route))
with self.assertRaises(FollowHistoryEnd):
connector.fetch("ffxivinitiala", page=1)
def test_a_fanbox_video_post_keeps_the_cover_as_its_thumbnail(self):
"""视频帖的缩略图退回列表封面,不拿视频地址当图。
FANBOX 的 `fileMap` 只给视频本体,没有 `thumbnailUrl`。把 mp4 地址填进
`thumb_url`,关注页那张 `
` 就指着一个视频文件——浏览器拉完几百兆也
画不出东西,卡片上是一片空白。
"""
def route(request):
return HttpResponse(200, {}, FANBOX_VIDEO_DETAIL_JSON if "post.info" in request.url
else FANBOX_JSON)
candidate = FanboxConnector(transport=_routed(route)).fetch("ffxivinitiala").candidates[0]
self.assertEqual(candidate.extra["media_items"][0]["media_kind"], "video")
self.assertIsNone(candidate.extra["media_items"][0]["thumb_url"])
self.assertEqual(candidate.thumb_url, "https://img.test/cover.jpg")
def test_fanbox_routes_only_post_info_through_the_browser_transport(self):
list_seen = []
detail_seen = []
def list_route(request):
list_seen.append(request)
return HttpResponse(200, {}, FANBOX_JSON)
def detail_route(request):
detail_seen.append(request)
return HttpResponse(200, {}, FANBOX_DETAIL_JSON)
result = FanboxConnector(
transport=_routed(list_route),
detail_transport=_routed(detail_route),
).fetch("ffxivinitiala")
self.assertEqual(len(result.candidates), 1)
self.assertTrue(all("post.listCreator" in request.url for request in list_seen))
self.assertEqual(len(detail_seen), 1)
self.assertIn("post.info", detail_seen[0].url)
self.assertEqual(USER_AGENT, detail_seen[0].headers["User-Agent"])
self.assertEqual(
detail_seen[0].headers["Referer"],
"https://www.fanbox.cc/@ffxivinitiala/posts/12489354",
)
def test_fanbox_uses_gofile_api_token_and_keeps_only_playable_files(self):
seen = []
def route(request):
seen.append(request)
if "api.gofile.io" in request.url:
return HttpResponse(200, {}, GOFILE_JSON)
return HttpResponse(200, {}, FANBOX_DETAIL_JSON if "post.info" in request.url
else FANBOX_JSON)
result = FanboxConnector(
transport=_routed(route),
gofile_credential=Credential("gofile", {"api_token": "secret"}),
).fetch("ffxivinitiala")
post = result.candidates[0]
self.assertEqual(post.extra["gofile_video_count"], 2)
self.assertEqual(len(post.extra["media_items"]), 4)
gofile_request = next(request for request in seen if "api.gofile.io" in request.url)
self.assertEqual(gofile_request.headers["Authorization"], "Bearer secret")
self.assertNotIn("secret", gofile_request.url)
def test_fanbox_cookie_stays_on_fanbox_requests(self):
seen = []
def route(request):
seen.append(request)
if "api.gofile.io" in request.url:
return HttpResponse(200, {}, GOFILE_JSON)
return HttpResponse(200, {}, FANBOX_DETAIL_JSON if "post.info" in request.url
else FANBOX_JSON)
FanboxConnector(
transport=_routed(route),
credential=Credential("fanbox", {"cookie": "FANBOXSESSID=session"}),
gofile_credential=Credential("gofile", {"api_token": "secret"}),
).fetch("ffxivinitiala")
fanbox_requests = [request for request in seen if "api.fanbox.cc" in request.url]
self.assertTrue(fanbox_requests)
self.assertTrue(all(request.headers.get("Cookie") == "FANBOXSESSID=session"
for request in fanbox_requests))
gofile_request = next(request for request in seen if "api.gofile.io" in request.url)
self.assertNotIn("Cookie", gofile_request.headers)
def test_gofile_premium_requirement_is_not_reported_as_a_bad_token(self):
def route(request):
if "api.gofile.io" in request.url:
return HttpResponse(401, {"content-type": "application/json"},
b'{"status":"error-notPremium","data":{}}')
return HttpResponse(200, {}, FANBOX_DETAIL_JSON if "post.info" in request.url
else FANBOX_JSON)
connector = FanboxConnector(
transport=_routed(route),
gofile_credential=Credential("gofile", {"api_token": "valid-token"}),
)
with self.assertRaises(FollowSourceError) as caught:
connector.fetch("ffxivinitiala")
self.assertIn("Premium", str(caught.exception))
self.assertIn("token 有效", str(caught.exception))
def test_one_blocked_detail_keeps_the_public_list_item(self):
def route(request):
if "post.info" in request.url:
return HttpResponse(403, {}, b"challenge")
return HttpResponse(200, {}, FANBOX_JSON)
result = FanboxConnector(transport=_routed(route)).fetch("ffxivinitiala")
self.assertEqual(len(result.candidates), 1)
self.assertIn("HTTP 403", result.candidates[0].extra["media_error"])
# 不写空清单:`partial` 行落库走 `json_patch`,补丁里的 null 是「删掉这个
# 键」,一篇临时被挡就会把上一轮取到的媒体清单抹掉。
self.assertNotIn("media_items", result.candidates[0].extra)
self.assertTrue(result.candidates[0].partial,
"详情没取到就不算补齐,下一轮还要再问一次")
def test_fanbox_asks_post_info_only_for_posts_it_has_not_read_yet(self):
"""已经补齐过的帖子不再打一次 post.info。
一页 10 条、每条一次详情,是这条来源里唯一会随条目数增长的请求。列表页的
304 只在**整页没变**时省下它们;新增一条就会连带把另外九条重新问一遍。
"""
seen = []
def route(request):
seen.append(request.url)
return HttpResponse(200, {}, FANBOX_DETAIL_JSON
if "post.info" in request.url else FANBOX_JSON)
result = FanboxConnector(transport=_routed(route),
enrich_skip={"12489354"}).fetch("ffxivinitiala")
self.assertEqual([url for url in seen if "post.info" in url], [])
self.assertEqual(result.probed, 0)
item = result.candidates[0]
self.assertTrue(item.partial, "跳过第二阶段的条目必须标 partial")
self.assertNotIn("media_items", item.extra,
"详情才知道的键不能带着 null 进补丁,那会把库里的媒体删掉")
# 列表本来就给得出的那几项仍然在:候选不是空壳。
self.assertEqual(item.url, "https://ffxivinitiala.fanbox.cc/posts/12489354")
self.assertEqual(item.published_at, "2026-08-26T14:34:51Z")
def test_fanbox_general_error_is_reported_instead_of_as_bad_json_shape(self):
def route(request):
if "post.info" in request.url:
return HttpResponse(200, {"content-type": "application/json"},
b'{"error":"general_error"}')
return HttpResponse(200, {}, FANBOX_JSON)
result = FanboxConnector(transport=_routed(route)).fetch("ffxivinitiala")
self.assertIn("general_error", result.candidates[0].extra["media_error"])
def test_subscribestar_reads_public_profile_posts_without_login(self):
result = SubscribeStarConnector(
transport=_transport(body=SUBSCRIBESTAR_HTML)).fetch(
"subscribestar.adult/initiala")
self.assertEqual(len(result.candidates), 1)
post = result.candidates[0]
self.assertEqual(post.external_id, "2650844")
self.assertEqual(post.title, "MobiusFF Sarah Animations")
self.assertEqual(post.author, "InitialA")
self.assertEqual(post.url, "https://subscribestar.adult/posts/2650844")
def test_patreon_uses_server_rendered_public_cards_and_deduplicates_links(self):
result = PatreonConnector(transport=_transport(body=PATREON_HTML)).fetch("sample")
self.assertEqual(len(result.candidates), 1)
self.assertEqual(result.candidates[0].external_id, "167576581")
self.assertEqual(result.candidates[0].title, "New public work")
self.assertEqual(result.candidates[0].group_hint, "patreon:167576581")
# 形状取自 2026-08-25 对 api.rule34.xxx 的实测响应:顶层是裸列表,`image` 是 32 位
# 十六进制哈希(15/15),`parent_id` 全是 0(15/15),`source` 13/15 有值。
# `tag_info` 是 `fields=tag_info` 加上的(2026-11-03 实测):普通标签的类型写作 `tag`,
# 名字和 `tags` 一样是 HTML 转义形态。
RULE34XXX_JSON = json.dumps([
{"id": 18534395, "image": "3df1cbc67e072d6144588d6c80e490ea.mp4", "parent_id": 0,
"tags": "lazyprocrastinator fiona blush video sound miqo'te", "change": 1787445373,
"tag_info": [
{"count": 120, "type": "artist", "tag": "lazyprocrastinator"},
{"count": 900, "type": "character", "tag": "fiona"},
{"count": 50000, "type": "tag", "tag": "blush"},
{"count": 80000, "type": "metadata", "tag": "video"},
{"count": 70000, "type": "metadata", "tag": "sound"},
{"count": 20796, "type": "character", "tag": "miqo'te"},
{"count": 3, "type": "deprecated", "tag": "old_name"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3df1.mp4",
"sample_url": "https://api-cdn.rule34.xxx/images/1232/3df1.jpg",
"preview_url": "https://api-cdn.rule34.xxx/thumbnails/1232/t_3df1.jpg",
"width": 1280, "height": 720,
"score": 72, "source": "https://lazyprocrast.fanbox.cc/posts/12304831"},
{"id": 18534396, "image": "3bda1572c365b223d8b287c538a38956.mp4", "parent_id": 0,
"tags": "lazyprocrastinator fiona video", "change": 1787445300,
"tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3bda.mp4",
"score": 70, "source": "https://www.fanbox.cc/@lazyprocrast/posts/12304831"},
{"id": 18534397, "image": "93eaeadff5effabe078c738aab85b3bc.mp4", "parent_id": 0,
"tags": "lazyprocrastinator sayuri video", "change": 1787445200,
"tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/93ea.mp4", "score": 12},
{"id": 18534398, "image": "fiona_paizuri_nude.mp4", "parent_id": 18534397,
"tags": "lazyprocrastinator fiona nude", "change": 1787445100,
"tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/fp.mp4", "score": 9},
]).encode()
PAHEAL_LIST_HTML = b""""""
PAHEAL_DETAIL_HTML = b"""
"""
def _routed(route):
"""按 URL 分派的测试传输。探测详情页的请求要能和列表请求分开回不同的响应。"""
def call(request, timeout, max_bytes):
return route(request)
return call
class Rule34PahealConnectorTests(unittest.TestCase):
def _connector(self):
return Rule34PahealConnector(transport=_routed(
lambda request: HttpResponse(
200, {}, PAHEAL_DETAIL_HTML if "/post/view/" in request.url
else PAHEAL_LIST_HTML)))
def test_tag_page_uses_detail_source_for_exact_cross_site_grouping(self):
result = self._connector().fetch("initiala")
self.assertEqual(len(result.candidates), 1)
item = result.candidates[0]
self.assertEqual(item.external_id, "7428820")
self.assertEqual(item.group_hint, "subscribestar:2639932")
self.assertEqual(item.duration, 28.4)
self.assertEqual(item.published_at, "2026-08-26T15:21:00Z")
self.assertEqual(item.media_url,
"https://r34i.paheal-cdn.net/df/fb/video")
self.assertFalse(item.title_is_name)
def test_list_thumb_media_kind_reads_extension_or_mime(self):
"""视频帖在图片视图里没有图片尺寸可占位,判成图片就混进瀑布流。"""
listing = b""""""
connector = Rule34PahealConnector(transport=_routed(
lambda request: HttpResponse(
200, {}, PAHEAL_DETAIL_HTML if "/post/view/" in request.url else listing)))
kinds = {item.external_id: item.extra["media_kind"]
for item in connector.fetch("initiala").candidates}
self.assertEqual(kinds, {"1": "video", "2": "video", "3": "image"})
def test_backfill_404_means_history_is_exhausted(self):
connector = Rule34PahealConnector(transport=_transport(status=404))
with self.assertRaises(FollowHistoryEnd):
connector.fetch("initiala", page=3)
def test_a_throttled_detail_page_is_retried_without_really_waiting(self):
"""列表一页 24 条、每条一次详情页,被按频率挡回来是常态。
退避真的 sleep 会让这条测试等满 7 秒,于是它一直没被写;没有测试的结果是
「挡回来一次就当这条没有上传时间」这种缺陷只能在生产数据里发现。
"""
attempts, waits = [], []
def route(request):
if "/post/view/" not in request.url:
return HttpResponse(200, {}, PAHEAL_LIST_HTML)
attempts.append(request.url)
if len(attempts) < 3:
return HttpResponse(429, {}, b"slow down")
return HttpResponse(200, {}, PAHEAL_DETAIL_HTML)
result = Rule34PahealConnector(transport=_routed(route),
sleeper=waits.append).fetch("initiala")
self.assertEqual(len(attempts), 3)
self.assertEqual(waits, [1.0, 2.0], "退避节奏是声明的那三档,按需取用")
item = result.candidates[0]
self.assertEqual(item.published_at, "2026-08-26T15:21:00Z")
self.assertFalse(item.partial)
def test_a_detail_page_that_stays_blocked_leaves_the_row_partial(self):
"""取不到就保持列表视图。落库时 partial 行不动上一轮取到的时间与时长。"""
waits = []
def route(request):
if "/post/view/" not in request.url:
return HttpResponse(200, {}, PAHEAL_LIST_HTML)
return HttpResponse(503, {}, b"nope")
result = Rule34PahealConnector(transport=_routed(route),
sleeper=waits.append).fetch("initiala")
item = result.candidates[0]
self.assertTrue(item.partial)
self.assertIsNone(item.published_at)
self.assertIsNone(item.duration)
# 列表页本来就给得出的那几项仍然在:候选不是空壳。
self.assertEqual(item.external_id, "7428820")
self.assertEqual(item.media_url, "https://r34i.paheal-cdn.net/df/fb/video")
self.assertEqual(item.group_hint, "rule34paheal:post:7428820")
self.assertEqual(waits, [1.0, 2.0, 4.0, 8.0])
def test_details_already_taken_cost_no_request_at_all(self):
"""第二阶段只打新条目。这是唯一会让请求数随条目数增长的路径。"""
seen = []
def route(request):
seen.append(request.url)
return HttpResponse(200, {}, PAHEAL_DETAIL_HTML
if "/post/view/" in request.url else PAHEAL_LIST_HTML)
result = Rule34PahealConnector(transport=_routed(route),
enrich_skip={"7428820"}).fetch("initiala")
self.assertEqual(len(seen), 1, "库里已经补齐过的条目不该再打一次详情页")
self.assertEqual(result.probed, 0)
item = result.candidates[0]
self.assertTrue(item.partial, "跳过第二阶段的条目必须标 partial")
self.assertNotIn("source", item.extra,
"详情才知道的键不能带着 null 进补丁,那会把库里的出处删掉")
class KemonoConnectorTests(unittest.TestCase):
def test_fetch_normalizes_posts_and_sends_the_documented_accept_header(self):
seen = []
connector = KemonoConnector(transport=_transport(body=KEMONO_POSTS, record=seen,
headers={"ETag": '"k1"'}))
result = connector.fetch("fanbox/30917150")
self.assertEqual(seen[0].url,
"https://kemono.cr/api/v1/fanbox/user/30917150/posts")
# 站点自己在 403 响应体里要求抓取带这个头,不是绕过防护。
self.assertEqual(seen[0].headers["Accept"], "text/css")
self.assertEqual(seen[0].headers["User-Agent"], USER_AGENT)
self.assertEqual(result.etag, '"k1"')
self.assertEqual(result.semantics, "work")
self.assertEqual(len(result.candidates), 2)
first = result.candidates[0]
self.assertEqual(first.external_id, "11406814")
self.assertEqual(first.published_at, "2026-02-14T21:51:10Z")
self.assertEqual(first.url,
"https://kemono.cr/fanbox/user/30917150/post/11406814")
# 交付文件取非图片的那个:夹具里 file 是 a.png、附件是 a.zip,压缩包才是正片。
# 按 file.path 优先会把整条判成图片——pawchive 上作者用 gif 当预览、mp4 放附件,
# 正是这条路径让两个 1080p 正片消失(2026-08-30 取证)。
# /data 前缀是必需的:少了它三站的原始文件都是 404(2026-08-30 取证)。
self.assertEqual(first.media_url, "https://kemono.cr/data/1c/fa/1cfae7.zip")
self.assertEqual(first.extra["attachment_count"], 1)
# post id 就是原平台的 post id,和别的站点从 source 归一出的键同一个命名空间。
self.assertEqual(first.group_hint, "fanbox:11406814")
def test_archive_posts_get_a_cover_thumbnail(self):
"""归档站的卡片必须带封面:不设 `thumb_url` 不是取不到,是压根没去取。
2026-08-30 实测缩略图走 `img.` 子域:主域 kemono.cr 回 302、pawchive.pw 回 404;
去掉 `thumbnail/` 前缀则是 404,所以这个前缀是必需的。
"""
result = KemonoConnector(transport=_transport(body=KEMONO_POSTS)).fetch("fanbox/1")
self.assertEqual(result.candidates[0].thumb_url,
"https://img.kemono.cr/thumbnail/data/1c/fa/1cfae7.png")
def test_archive_video_uses_an_image_attachment_as_its_cover(self):
"""主资源是视频/压缩包时,后附的图片仍然是可用封面。"""
posts = [{
"id": "cover-after-video", "title": "Animation pack",
"published": "2026-08-27T00:00:00Z",
"file": {"path": "/video/release.mp4"},
"attachments": [
{"path": "/video/notes.txt"},
{"path": "/cover/release.webp"},
],
}]
result = KemonoConnector(
transport=_transport(body=json.dumps(posts).encode())).fetch("fanbox/1")
candidate = result.candidates[0]
self.assertEqual(candidate.media_url, "https://kemono.cr/data/video/release.mp4")
self.assertEqual(candidate.thumb_url,
"https://img.kemono.cr/thumbnail/data/cover/release.webp")
def test_archive_posts_with_several_media_list_them_for_the_detail_carousel(self):
"""两张以上图或视频的帖子把它们列进媒体清单,详情里才翻得到第二张。
交付文件排第一(条目级媒体类型跟清单首项走),其余按帖子顺序;`file` 在附件里
重复列出的按路径去重;文本、压缩包不进清单。只有一张的帖子不建清单。
"""
posts = [
{"id": "multi", "title": "Pack", "published": "2026-08-27T00:00:00Z",
"file": {"name": "cover.gif", "path": "/a/cover.gif"},
"attachments": [{"name": "cover.gif", "path": "/a/cover.gif"},
{"name": "clip.mp4", "path": "/a/clip.mp4"},
{"name": "notes.txt", "path": "/a/notes.txt"},
{"name": "page2.jpg", "path": "/a/page2.jpg"}]},
{"id": "single", "title": "One", "published": "2026-08-26T00:00:00Z",
"file": {"name": "only.png", "path": "/b/only.png"},
"attachments": [{"name": "only.zip", "path": "/b/only.zip"}]},
]
result = KemonoConnector("pawchive", transport=_transport(
body=json.dumps(posts).encode())).fetch("patreon/1")
multi, single = result.candidates
self.assertEqual(multi.media_url, "https://file.pawchive.pw/data/a/clip.mp4")
items = multi.extra["media_items"]
self.assertEqual([(media["id"], media["media_kind"]) for media in items],
[("/a/clip.mp4", "video"), ("/a/cover.gif", "image"),
("/a/page2.jpg", "image")])
self.assertEqual(items[1], {
"id": "/a/cover.gif", "name": "cover.gif",
"url": "https://file.pawchive.pw/data/a/cover.gif",
"thumb_url": "https://img.pawchive.pw/thumbnail/data/a/cover.gif",
"media_kind": "image", "resource_provider": "pawchive"})
self.assertIsNone(items[0]["thumb_url"])
# 卡面封面与清单里第一张图是同一张,界面据此沿用条目级量过的比例。
self.assertEqual(multi.thumb_url, items[1]["thumb_url"])
self.assertNotIn("media_items", single.extra)
def test_no_thumbnail_is_offered_for_things_that_have_none(self):
# 视频和压缩包没有缩略图,给了也是 404——那会让卡片显示一张碎图,
# 比一个干净的占位更糟。
connector = KemonoConnector()
self.assertIsNone(connector._thumb_url("/a/b/clip.mp4"))
self.assertIsNone(connector._thumb_url("/a/b/pack.zip"))
self.assertIsNone(connector._thumb_url(None))
def test_a_post_that_only_links_a_file_host_is_a_release(self):
"""用户定的判据:资源贴要么贴附件,要么附上网盘链接。
只做图的作者也算——一张图片附件就够,不要求压缩包或视频。
"""
posts = json.loads(KEMONO_POSTS)
posts.append({"id": "777", "title": "Monthly pack",
"published": "2026-02-01T00:00:00", "file": {}, "attachments": [],
"substring": "gofile - https://gofile.io/d/xyz"})
result = KemonoConnector(
transport=_transport(body=json.dumps(posts).encode())).fetch("fanbox/1")
self.assertIn("777", [c.external_id for c in result.candidates])
self.assertEqual(result.skipped, 0)
self.assertEqual(result.probed, 0, "摘要里就能判出来,不该多打一次站点")
def test_a_post_it_cannot_judge_is_probed_not_guessed(self):
"""列表判不出来时去抓详情,而不是猜。
列表接口只给 `substring`(正文摘要),网盘链接常常在摘要之外。判不出来直接
当「不是 release」会删掉真东西,当「是」又等于没过滤——所以判据是三态,
只有第三种答案才允许下一步联网。
"""
posts = json.loads(KEMONO_POSTS)
posts.append({"id": "888", "title": "March pack", "published": "2026-03-01T00:00:00",
"file": {}, "attachments": [], "substring": "Here you go"})
detail = json.dumps({"post": {"id": "888",
"content": "https://mega.nz/folder/abc
"}}).encode()
bodies = {"posts": json.dumps(posts).encode(), "post/888": detail}
def route(request):
for key, body in bodies.items():
if request.url.endswith(key):
return HttpResponse(200, {}, body)
raise AssertionError(f"未预期的请求:{request.url}")
result = KemonoConnector(transport=_routed(route)).fetch("fanbox/1")
self.assertIn("888", [c.external_id for c in result.candidates])
self.assertEqual(result.probed, 1)
def test_a_post_already_in_the_ledger_is_not_probed_again(self):
"""库里有的帖子当初判过保留;每次检查都再探一遍,请求数就跟着库存涨。"""
posts = json.loads(KEMONO_POSTS)
posts.append({"id": "888", "title": "March pack", "published": "2026-03-01T00:00:00",
"file": {}, "attachments": [], "substring": "Here you go"})
def route(request):
if "/post/" in request.url:
raise AssertionError(f"未预期的详情请求:{request.url}")
return HttpResponse(200, {}, json.dumps(posts).encode())
result = KemonoConnector(transport=_routed(route), enrich_skip={"888"}).fetch("fanbox/1")
self.assertIn("888", [c.external_id for c in result.candidates])
self.assertEqual(result.probed, 0)
def test_a_rate_limited_probe_stops_the_check(self):
"""探测被 429 挡回来时不能当成「保留」接着探下一帖,要让这次检查停下。"""
posts = json.loads(KEMONO_POSTS)
posts.extend({"id": str(ident), "title": f"pack {ident}",
"published": "2026-03-01T00:00:00", "file": {}, "attachments": [],
"substring": "Here"} for ident in (801, 802, 803))
probes = []
def route(request):
if "/post/" in request.url:
probes.append(request.url)
return HttpResponse(429, {"Retry-After": "90"}, b"")
return HttpResponse(200, {}, json.dumps(posts).encode())
with self.assertRaises(FollowSourceRateLimited) as raised:
KemonoConnector(transport=_routed(route)).fetch("fanbox/1")
self.assertEqual(raised.exception.retry_after, 90.0)
self.assertEqual(len(probes), 1)
def test_a_post_with_no_resource_at_all_is_dropped(self):
"""公告、感谢这类帖子既没有附件也没有网盘链接,抓了详情之后确认丢掉。"""
posts = json.loads(KEMONO_POSTS)
posts.append({"id": "999", "title": "Thanks for 10k followers!",
"published": "2026-02-01T00:00:00", "file": {}, "attachments": [],
"substring": "You are all wonderful"})
detail = json.dumps({"post": {"id": "999",
"content": "See you next month
"}}).encode()
def route(request):
if request.url.endswith("post/999"):
return HttpResponse(200, {}, detail)
return HttpResponse(200, {}, json.dumps(posts).encode())
result = KemonoConnector(transport=_routed(route)).fetch("fanbox/1")
self.assertEqual(result.skipped, 1)
self.assertNotIn("999", [c.external_id for c in result.candidates])
def test_an_unreachable_probe_keeps_the_post(self):
"""抓不到详情就保留。
网络抖一下就删掉用户的一份更新,是拿一次失败的请求换一次不可见的数据丢失。
探测额度用完时同理——额度是内部限额,不该变成删数据的理由。
"""
posts = json.loads(KEMONO_POSTS)
posts.append({"id": "555", "title": "April pack", "published": "2026-04-01T00:00:00",
"file": {}, "attachments": [], "substring": "Here"})
def route(request):
if "post/555" in request.url:
return HttpResponse(503, {}, b"")
return HttpResponse(200, {}, json.dumps(posts).encode())
waits = []
result = KemonoConnector(transport=_routed(route), sleeper=waits.append).fetch("fanbox/1")
self.assertIn("555", [c.external_id for c in result.candidates])
self.assertEqual(waits, [1, 2, 4, 8])
# 额度为 0 时不联网,也不删。
offline = KemonoConnector(max_probes=0,
transport=_transport(body=json.dumps(posts).encode()))
kept = offline.fetch("fanbox/1")
self.assertIn("555", [c.external_id for c in kept.candidates])
self.assertEqual(kept.probed, 0)
def test_a_release_named_after_a_poll_is_still_a_release(self):
"""**不要按标题关键词丢投票贴。**
2026-08-27 拿 LazyProcrastinator 的真实 50 条跑过一版
`poll|vote|survey|…` 正则,丢掉 18 条,其中包括
`Public Poll Release + Littlest Ramble`、`October Poll Animations Released`
——这位作者的正片就是按投票结果命名的。列表接口不给帖子类型字段,
标题不足以区分「投票贴」和「投票选出的成品」,误删一份 release
比多出一张卡片糟糕得多。这条测试就是为了挡住那个正则再被加回来。
"""
posts = json.loads(KEMONO_POSTS)
posts[0] = {**posts[0], "id": "777",
"title": "Public Poll Release + Littlest Ramble"}
result = KemonoConnector(
transport=_transport(body=json.dumps(posts).encode())).fetch("fanbox/1")
self.assertIn("777", [c.external_id for c in result.candidates])
self.assertEqual(result.skipped, 0)
def test_paging_back_uses_an_offset_and_drops_the_conditional_headers(self):
"""往回翻要换页,而且**不能带条件请求头**。
`If-None-Match` 里存的是第一页的 etag。拿它去问第二页,站点很可能回 304,
用户点了「抓更早的」却什么都没发生——而且看起来和「确实没有更早的了」
一模一样。
2026-08-27 实测:kemono 的列表接口一页 50 条,`?o=50` 拿到的是第 51 条起,
与第一页零重叠。
"""
seen = []
connector = KemonoConnector(
transport=_transport(body=KEMONO_POSTS, record=seen))
connector.fetch("fanbox/30917150", etag='"k1"', last_modified="then", page=2)
self.assertEqual(seen[0].url,
"https://kemono.cr/api/v1/fanbox/user/30917150/posts?o=100")
self.assertNotIn("If-None-Match", seen[0].headers)
self.assertNotIn("If-Modified-Since", seen[0].headers)
def test_the_first_page_still_sends_conditional_headers(self):
# 常规检查必须保留条件请求:追更每天都在跑,没变就不该把整页再传一遍。
seen = []
KemonoConnector(transport=_transport(body=KEMONO_POSTS, record=seen)).fetch(
"fanbox/30917150", etag='"k1"')
self.assertEqual(seen[0].headers.get("If-None-Match"), '"k1"')
self.assertNotIn("?o=", seen[0].url)
def test_pawchive_returns_a_bare_list_and_uses_its_own_host(self):
seen = []
connector = KemonoConnector(provider="pawchive",
transport=_transport(body=KEMONO_POSTS, record=seen))
result = connector.fetch("fanbox/30917150")
self.assertTrue(seen[0].url.startswith("https://pawchive.pw/"))
self.assertEqual(result.provider, "pawchive")
self.assertEqual(len(result.candidates), 2)
def test_dict_shaped_payload_is_accepted(self):
body = json.dumps({"posts": json.loads(KEMONO_POSTS)}).encode()
result = KemonoConnector(transport=_transport(body=body)).fetch("fanbox/1")
self.assertEqual(len(result.candidates), 2)
def test_max_items_bounds_the_result(self):
connector = KemonoConnector(max_items=1, transport=_transport(body=KEMONO_POSTS))
self.assertEqual(len(connector.fetch("fanbox/1").candidates), 1)
def test_not_modified_returns_no_candidates(self):
connector = KemonoConnector(transport=_transport(status=304))
result = connector.fetch("fanbox/1", etag='"k1"')
self.assertTrue(result.not_modified)
self.assertEqual(result.candidates, ())
def test_backfill_400_means_history_is_exhausted(self):
connector = KemonoConnector(transport=_transport(status=400))
with self.assertRaises(FollowHistoryEnd):
connector.fetch("fanbox/1", page=2)
def test_malformed_ref_is_rejected_before_any_request(self):
seen = []
connector = KemonoConnector(transport=_transport(record=seen))
for bad in ("30917150", "fanbox/", "../etc/passwd", "fanbox/a b"):
with self.assertRaises(FollowSourceError):
connector.fetch(bad)
self.assertEqual(seen, [])
def test_forbidden_status_names_credentials_or_bot_check(self):
connector = KemonoConnector(transport=_transport(status=403))
with self.assertRaises(FollowSourceError) as caught:
connector.fetch("fanbox/1")
self.assertIn("403", str(caught.exception))
def test_unknown_kemono_host_is_rejected(self):
with self.assertRaises(FollowSourceError):
KemonoConnector(provider="nope")
class Rule34VideoConnectorTests(unittest.TestCase):
def _fetch(self, body=RULE34VIDEO_HTML, **kwargs):
return Rule34VideoConnector(transport=_transport(body=body), **kwargs).fetch(
"lazyprocrastinator")
def test_creator_page_yields_deduplicated_videos_with_duration(self):
result = self._fetch()
self.assertEqual([c.external_id for c in result.candidates],
["4542721", "4542713"])
self.assertEqual(result.candidates[0].duration, 20.0)
self.assertEqual(result.candidates[1].duration, 433.0)
def test_empty_backfill_page_means_history_is_exhausted(self):
connector = Rule34VideoConnector(transport=_transport(body=b""))
with self.assertRaises(FollowHistoryEnd):
connector.fetch("lazyprocrastinator", page=4)
def test_data_uri_placeholder_is_not_used_as_a_thumbnail(self):
first = self._fetch().candidates[0]
self.assertTrue(first.thumb_url.endswith("/3.jpg"))
self.assertTrue(first.media_url.endswith("_preview.mp4/"))
def test_relative_dates_are_marked_approximate(self):
first = self._fetch().candidates[0]
self.assertEqual(first.extra["published_precision"], "approximate")
self.assertEqual(first.extra["added_text"], "1 week ago")
self.assertIsNotNone(first.published_at)
def test_missing_dates_are_reported_as_unknown_not_guessed(self):
body = b''
candidate = self._fetch(body=body).candidates[0]
self.assertIsNone(candidate.published_at)
self.assertEqual(candidate.extra["published_precision"], "unknown")
def test_empty_parse_is_an_error_not_an_empty_success(self):
# 结构变了却报「本次没有更新」,会把站点改版静默吞掉。
with self.assertRaises(FollowSourceError):
self._fetch(body=b"no videos here")
def test_slug_is_validated(self):
connector = Rule34VideoConnector(transport=_transport(body=RULE34VIDEO_HTML))
with self.assertRaises(FollowSourceError):
connector.fetch("../models")
def test_detail_enrichment_adds_full_cover_video_date_and_content_tags(self):
seen = []
def transport(request, timeout, max_bytes):
seen.append(request.url)
body = (RULE34VIDEO_HTML if "/models/" in request.url
else RULE34VIDEO_DETAIL_HTML)
return HttpResponse(200, {}, body)
result = Rule34VideoConnector(transport=transport).fetch("lazyprocrastinator")
first = result.candidates[0]
self.assertEqual(result.probed, 2)
self.assertEqual(first.published_at, "2026-08-18T00:00:00Z")
self.assertEqual(first.extra["published_precision"], "exact")
self.assertEqual(first.duration, 433.0)
self.assertEqual(first.extra["tags"], ["deep throat", "breast squeeze"])
self.assertEqual(first.extra["categories"], ["3D", "Final Fantasy"])
self.assertEqual(first.extra["tag_types"]["3D"], "metadata")
self.assertEqual(first.extra["tag_types"]["Final Fantasy"], "copyright")
self.assertTrue(first.thumb_url.endswith("/preview.jpg"))
self.assertIn("4542721_360.mp4", first.media_url)
def test_characters_follow_the_site_group_not_the_name_shape(self):
"""角色按站方的 Characters 分组记 character,名字带括号不算依据。
`d.mon (overwatch)` 这类标签站方列在 Tags 组里,照记 general。
"""
groups = (
b''
b'')
detail = RULE34VIDEO_DETAIL_HTML.replace(b"