#!/usr/bin/env python3
"""
bailii_fetch.py — Search BAILII and download a UK/Irish case as a clean PDF.
BAILII now sits behind Anubis, a JavaScript proof-of-work bot wall, and re-issues
challenges on most requests. This script solves the proof-of-work in pure Python
(no browser needed), handles the quirks of BAILII's CGI endpoints, strips the page
chrome, and renders the judgment to PDF.
Usage:
python3 bailii_fetch.py cite "[2024] EAT 114" [--out DIR] [--keep-html] [--no-pdf]
python3 bailii_fetch.py url "https://www.bailii.org/uk/cases/UKEAT/2024/114.html" [--out DIR]
python3 bailii_fetch.py search "Uber Aslam" [--limit 10]
Notes:
* `cite` — most reliable. Give a neutral citation; BAILII maps it to the case URL.
* `url` — when you already have the BAILII case URL.
* `search`— name-based; prints candidates (it does NOT auto-download, because
BAILII's title search is fuzzy and you should choose the right hit).
* Only the Python standard library is required for fetching. PDF rendering uses
whichever of these is found: wkhtmltopdf, Chrome/Chromium headless, or weasyprint.
If none is present the cleaned HTML is saved and the path reported.
"""
import sys, os, re, json, time, argparse, hashlib, subprocess, shutil
import urllib.parse, urllib.request, urllib.error, http.cookiejar
import html as H
BASE = "https://www.bailii.org"
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36")
# --------------------------------------------------------------------------- #
# HTTP layer (Anubis-aware) #
# --------------------------------------------------------------------------- #
class _NoRedirect(urllib.request.HTTPRedirectHandler):
# BAILII's citation finder 302-redirects to the case URL, and the redirect
# target re-challenges. We follow redirects manually so each hop can solve
# its own challenge, and so we can percent-encode targets containing spaces.
def redirect_request(self, *a):
return None
def make_opener():
cj = http.cookiejar.CookieJar()
op = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(cj), _NoRedirect())
op.addheaders = [("User-Agent", UA),
("Accept", "text/html,application/xhtml+xml,*/*;q=0.8"),
("Accept-Language", "en-GB,en;q=0.9")]
return op
def _raw(op, url, data=None, timeout=45):
req = urllib.request.Request(url, data=data)
try:
with op.open(req, timeout=timeout) as r:
return r.read().decode("utf-8", "replace"), r.geturl(), r.getcode(), dict(r.headers)
except urllib.error.HTTPError as e:
return e.read().decode("utf-8", "replace"), url, e.code, dict(e.headers)
def is_anubis(text):
return ("anubis_challenge" in text) or ("Making sure you" in text)
def _solve_anubis(op, page_html):
"""Solve the Anubis proof-of-work and redeem it for the auth cookie.
Anubis "fast": find nonce so SHA256(randomData + nonce) has `difficulty`
leading zero hex chars, then GET the pass-challenge endpoint. We redeem with
redir='/' (a harmless GET) — never the original POST endpoint, which 500s.
"""
m = re.search(r'', page_html, re.S)
if not m:
return
ch = json.loads(m.group(1).strip())["challenge"]
rand, diff, cid = ch["randomData"], int(ch.get("difficulty", 2)), ch["id"]
prefix, nonce, t0 = "0" * diff, 0, time.time()
while True:
digest = hashlib.sha256((rand + str(nonce)).encode()).hexdigest()
if digest.startswith(prefix):
break
nonce += 1
qs = urllib.parse.urlencode({
"id": cid, "response": digest, "nonce": nonce,
"redir": "/", "elapsedTime": int((time.time() - t0) * 1000)})
_raw(op, f"{BASE}/.within.website/x/cmd/anubis/api/pass-challenge?{qs}")
def request(op, url, data=None, tries=12, follow=True):
"""Fetch a URL, transparently solving Anubis and following redirects.
Anubis re-challenges frequently even with a valid cookie, so we solve on
every challenged response and retry the same request. Returns (body, final_url, code).
"""
body = final = ""
code = 0
for _ in range(tries):
body, final, code, hdrs = _raw(op, url, data=data)
if is_anubis(body):
_solve_anubis(op, body)
continue
if code >= 500:
time.sleep(0.4)
continue
loc = next((v for k, v in hdrs.items() if k.lower() == "location"), None)
if follow and code in (301, 302, 303, 307, 308) and loc:
# percent-encode spaces (BAILII's search redirect contains them)
url = urllib.parse.urljoin(url, loc).replace(" ", "%20")
data = None
continue
return body, final, code
return body, final, code
def warm(op):
"""Obtain the first Anubis cookie via a harmless homepage GET."""
request(op, BASE + "/")
def fetch_case(op, url, attempts=5):
"""Fetch a case page, validating it is a genuine judgment (not a stray
challenge or error page). Retries the whole fetch if validation fails."""
body, final = "", url
for _ in range(attempts):
body, final, code = request(op, url)
if (code == 200 and not is_anubis(body) and len(body) > 8000
and "/cases/" in final and final.endswith(".html")):
return body, final
time.sleep(0.3)
return body, final
# --------------------------------------------------------------------------- #
# Search / resolve #
# --------------------------------------------------------------------------- #
def resolve_citation(op, citation):
"""Map a neutral citation to its BAILII case URL + HTML. Returns (url, html)
or (None, html) when BAILII has no direct match."""
body, final, code = request(
op, BASE + "/cgi-bin/find_by_citation.cgi",
data=urllib.parse.urlencode({"citation": citation}).encode())
if ("/cases/" in final and final.endswith(".html")
and not is_anubis(body) and len(body) > 8000):
# ensure we actually have the full case (re-fetch with validation)
body, final = fetch_case(op, final)
return final, body
return None, body
def search_by_name(op, name, limit=10):
"""Title-scoped search. BAILII's default all-fields search ranks alphabetically
and rarely surfaces the target case; restricting to the title field works far
better. Returns a list of (url, title) candidates."""
query = "title:(%s)" % " ".join(re.findall(r"[\w']+", name))
url = BASE + "/cgi-bin/lucy_search_1.cgi?" + urllib.parse.urlencode(
{"query": query, "mask_path": "/", "method": "boolean"})
body, final, code = request(op, url)
out, seen = [], set()
for href, txt in re.findall(
r']*>(.*?)', body, re.I | re.S):
if href in seen:
continue
seen.add(href)
title = re.sub(r"<[^>]+>|\s+", " ", H.unescape(txt)).strip().lstrip(">").strip()
out.append((BASE + href, title))
if len(out) >= limit:
break
return out
# --------------------------------------------------------------------------- #
# Clean + render #
# --------------------------------------------------------------------------- #
def clean_html(raw, src_url):
"""Strip BAILII chrome (nav, search form, donation banner, footer, scripts,
images) and return (case_title, standalone_html)."""
mt = re.search(r"(.*?)", raw, re.S | re.I)
title = re.sub(r"\s+", " ", H.unescape(mt.group(1))).strip() if mt else "BAILII case"
h = raw
for pat in (r"", r"",
r"", r"<(iframe|object|embed)\b.*?",
r""):
h = re.sub(pat, "", h, flags=re.I | re.S)
h = re.sub(r"]*>", "", h, flags=re.I)
h = re.sub(r"]*>", "", h, flags=re.I)
# cut the footer (everything from the BAILII copyright line onward).
# Tag-tolerant: in the HTML there is usually an between "BAILII:" and
# "Copyright Policy", so a plain-text match would miss it.
h = re.split(r"BAILII:\s*(?:<[^>]+>\s*)*Copyright Policy", h,
maxsplit=1, flags=re.I)[0]
# remove top nav bar ("[ Home ] [ Databases ] ...") up to the case heading/rule
h = re.sub(r"\[\s*]*>\s*Home\s*\s*\].*?(?=]*>\s*New search\s*\s*\].*?"
r"\[\s*]*>\s*Help\s*\s*\]", "",
h, flags=re.I | re.S, count=1)
# remove the donation appeal block
h = re.sub(r"THE FUTURE OF BAILII.*?(?=]*>", h, maxsplit=1, flags=re.I)
inner = parts[1] if len(parts) > 1 else h
inner = re.sub(r".*", "", inner, flags=re.I | re.S)
css = ("")
banner = (""
"Retrieved from BAILII · %s
" % H.escape(src_url))
doc = (" %s%s%s