#!/usr/bin/env python3
"""bin/owngenome_live.py and assets/live/ on the synthetic DEMO-001 run (tests/fixtures/live).
Checks:
1. hash-password: the hash verifies the password and no other; the server
reads only a hash; a non-loopback --bind without one exits 2;
2. login: no cookie gives a redirect to /login (pages) or 401 (data); a wrong
password is 401 with a login-fail row; the right one sets an HttpOnly,
SameSite=Strict cookie; an expired session is 401; after 5 failures the
6th try is 429; the plain password is in no log, no stderr and no stored
message; a negative or non-numeric Content-Length on /login is 413;
3. Host: a foreign Host header is 403, an --allowed-host name is served;
4. ingest: POST /ingest on the browser port is 404; a 2 MiB body is 413,
a negative Content-Length 413, a body that is not JSON 400, a run named
'../x' 400; metadata.parameters
(an HPO list, a VCF path) and a task's script are not stored;
5. the event stream sends a posted message within 2 s, Last-Event-ID
resumes after the id given, what it sends equals messages(), and an open
stream ends at logout;
6. state.json's step states are run_state's for the same files; a run is
finished only while pgp_run_finished is its newest message; ClinVar's
headline is never 'ok' and carries the report's caveat;
7. every browser response carries the Content-Security-Policy, and
assets/live/ holds no outside URL, no inline script and no on* attribute;
8. a plugin (tests/fixtures/live/plugins) adds a flag, a route, a run file
and a state hook, all behind the session; the page gets one script tag
per file in assets/live/plugins;
9. run files: a file the report links to is served, an unlinked one and a
symlink out of the root are 404;
10. in a headless Chrome (skipped without Chrome, except in CI): the login
form, the step map, a box that repaints after a posted message, the
replay mode playing tests/fixtures/live/frames.json (opened at its last,
finished frame, its time short with the full time in the title), and no
policy violation;
11. with --tls-cert, a client that connects and sends nothing does not hold
up another client's HTTPS request.
Checks 2, 4, 5, 6, 7, 8 and 11 are also run against a planted mistake and must fail
there: a check that cannot fail is no check.
Run: python3 tests/test_owngenome_live.py
"""
import contextlib
import http.client
import io
import json
import os
import re
import select
import shutil
import socket
import subprocess
import sys
import tempfile
import threading
import time
import urllib.parse
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(REPO, "bin"))
import owngenome_live # noqa: E402
import run_state # noqa: E402
FIX = os.path.join(REPO, "tests", "fixtures", "live")
ASSETS = os.path.join(REPO, "assets", "live")
RUN = "DEMO-001"
PASSWORD = "synthetic test password 7"
FAILS = []
RESPONSES = [] # (path, headers) of every browser response
def check(desc, ok, detail=""):
print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:800]}"))
if not ok:
FAILS.append(desc)
def control(desc, still_passes):
"""A planted mistake: the check must fail on it."""
check(f"control: {desc}", not still_passes, "the check passed on the planted mistake")
def request(port, method, path, body=None, headers=None, record=True):
c = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
h = {"Host": f"127.0.0.1:{port}"}
h.update(headers or {})
c.request(method, path, body=body, headers=h)
r = c.getresponse()
data = r.read()
hdr = {k.lower(): v for k, v in r.getheaders()}
c.close()
if record:
RESPONSES.append((path, hdr))
return r.status, hdr, data
def make_root(work):
root = os.path.join(work, "genome")
d = os.path.join(root, RUN)
shutil.copytree(os.path.join(FIX, RUN), d)
os.makedirs(os.path.join(d, "clinvar"))
os.makedirs(os.path.join(d, "extra"))
with open(os.path.join(d, "clinvar", "hits.tsv"), "w") as f:
f.write("gene\tvariant\nGENEA\tchr1:100 A>G\n")
with open(os.path.join(d, "clinvar", "unlinked.tsv"), "w") as f:
f.write("not linked\n")
with open(os.path.join(d, "extra", "note.txt"), "w") as f:
f.write("plugin file\n")
outside = os.path.join(work, "outside.txt")
with open(outside, "w") as f:
f.write("outside the root\n")
os.symlink(outside, os.path.join(d, "clinvar", "escape.tsv"))
# The report links CPSR's folder, which holds its HTML report and its table
os.makedirs(os.path.join(d, "cpsr"))
with open(os.path.join(d, "cpsr", f"{RUN}.cpsr.grch38.html"), "w") as f:
f.write("
CPSR\n")
with open(os.path.join(d, "cpsr", f"{RUN}.cpsr.grch38.classification.tsv.gz"), "w") as f:
f.write("not linked\n")
os.makedirs(os.path.join(d, "twopages"))
for n in ("a.html", "b.html"):
with open(os.path.join(d, "twopages", n), "w") as f:
f.write("\n")
with open(os.path.join(d, f"{RUN}_report.html"), "w") as f:
f.write(''
'hits escape '
'the cpsr folder two pages '
'outside\n')
return root
def serve(argv, app=None):
live, web, ingest = owngenome_live.start(argv, app)
for s in (web, ingest):
threading.Thread(target=s.serve_forever, daemon=True).start()
return live, web, ingest
def stop(*servers):
for live, web, ingest in servers:
live.stopping = True
web.shutdown()
ingest.shutdown()
web.server_close()
ingest.server_close()
def login(port, password=PASSWORD):
code, h, _ = request(port, "POST", "/login", urllib.parse.urlencode({"password": password, "next": "/"}),
{"Content-Type": "application/x-www-form-urlencoded"})
m = re.match(r"owngenome_live=([^;]+)", h.get("set-cookie", ""))
return code, h, (m.group(1) if m else None)
def post(port, run, ev):
body = ev if isinstance(ev, (bytes, str)) else json.dumps(ev)
return request(port, "POST", f"/ingest/{run}", body, {"Content-Type": "application/json"}, record=False)
class Stream:
"""An open /events stream, read in a thread."""
def __init__(self, port, cookie, last=None):
self.closed = False
self.items, self.sock = [], socket.create_connection(("127.0.0.1", port), timeout=30)
hdr = f"GET /runs/{RUN}/events HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nCookie: owngenome_live={cookie}\r\n"
if last is not None:
hdr += f"Last-Event-ID: {last}\r\n"
self.sock.sendall((hdr + "\r\n").encode())
threading.Thread(target=self.read, daemon=True).start()
def read(self):
buf, cur = b"", {}
try:
while True:
chunk = self.sock.recv(65536)
if not chunk:
self.closed = True
return
buf += chunk
while b"\n" in buf:
line, buf = buf.split(b"\n", 1)
line = line.decode().rstrip("\r")
if line.startswith("id: "):
cur["id"] = int(line[4:])
elif line.startswith("data: "):
cur["data"] = json.loads(line[6:])
elif line == "" and "data" in cur:
self.items.append((cur["id"], cur["data"]))
cur = {}
except OSError:
return
def wait(self, n, timeout):
end = time.time() + timeout
while time.time() < end and len(self.items) < n:
time.sleep(0.02)
return len(self.items) >= n
def close(self):
try:
self.sock.close()
except OSError:
pass
def raw(port, head, body=b"", half_close=True, timeout=10):
"""Send a hand-made request and return the status line ('' when the server sent nothing)."""
sk = socket.create_connection(("127.0.0.1", port), timeout=timeout)
try:
sk.sendall(head.encode() + body)
if half_close:
sk.shutdown(socket.SHUT_WR) # a server that reads to the end returns instead of hanging
return sk.recv(200).decode(errors="replace").split("\r\n", 1)[0]
except OSError as e:
return f"error {type(e).__name__}"
finally:
sk.close()
def tls_checks(root, work, pwfile):
"""11. One silent client must not stop the HTTPS listener. Returns (ok, detail) for a given
wrap: the real server, and a planted one that shakes hands in accept()."""
cert, key = os.path.join(work, "c.pem"), os.path.join(work, "k.pem")
if not shutil.which("openssl"):
ci = os.environ.get("CI") == "true"
print(("[FAIL] " if ci else "[SKIP] ") + "no openssl to make a test certificate")
if ci:
FAILS.append("no openssl in CI")
return
r = subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", key, "-out", cert,
"-days", "1", "-subj", "/CN=localhost"], capture_output=True, text=True, timeout=120)
if r.returncode != 0:
check("openssl makes a test certificate", False, r.stderr)
return
def one(planted):
argv = ["--root", root, "--state-dir", os.path.join(work, "state-tls"), "--port", "0", "--ingest-port", "0",
"--password-file", pwfile, "--tls-cert", cert, "--tls-key", key, "--quiet"]
real_wrap = owngenome_live.ssl.SSLContext.wrap_socket
if planted:
def wrap(self, sock, **kw):
kw.pop("do_handshake_on_connect", None)
return real_wrap(self, sock, **kw)
owngenome_live.ssl.SSLContext.wrap_socket = wrap
try:
with contextlib.redirect_stderr(io.StringIO()):
srv = serve(argv)
finally:
owngenome_live.ssl.SSLContext.wrap_socket = real_wrap
live, web, ingest = srv
port = web.server_address[1]
silent = socket.create_connection(("127.0.0.1", port), timeout=10)
time.sleep(0.3) # let accept() take the silent one first
ctx = owngenome_live.ssl.create_default_context()
ctx.check_hostname, ctx.verify_mode = False, owngenome_live.ssl.CERT_NONE
try:
c = http.client.HTTPSConnection("127.0.0.1", port, timeout=3, context=ctx)
c.request("GET", "/login", headers={"Host": f"127.0.0.1:{port}"})
code = c.getresponse().status
c.close()
except OSError as e:
code = type(e).__name__
silent.close()
stop(srv)
return code
code = one(False)
check("with --tls-cert, a silent client does not hold up another's HTTPS request", code == 200, code)
control("the handshake done in accept()", one(True) == 200)
def asset_problems(texts):
"""(file, problem) for each outside URL, inline script or on* attribute in assets/live."""
out = []
for name, t in texts.items():
for u in re.findall(r"(?:https?:)?//[A-Za-z0-9.-]+\.[a-z]{2,}[^\s'\"]*", t):
if u != "http://www.w3.org/2000/svg": # the SVG namespace name, never fetched
out.append((name, u))
if name.endswith(".html"):
for tag in re.findall(r"