#!/usr/bin/env python3 """make_demo_sample.py: write DEMO-001, the invented sample of the docs pictures. make_demo_sample.py --genome-dir DIR [--seed 1] [--records 4700000] [--sample DEMO-001] [--sex male] make_demo_sample.py check --genome-dir DIR The first form writes DIR/DEMO-001 with every file steps 24, 27 and 28 read, as the steps that make them would have written them, so that every card of the reports shows a value. Nothing is read from a real sample: the values come from a random generator with a fixed seed and from a few hand-picked numbers below. It also writes the three reference files the run manifest reads (DIR/clinvar, DIR/t1k_idx, DIR/prs_scores), cut down to their header lines. Borrowed, not rewritten: - the PharmCAT report starts from PharmCAT's own example report (tests/fixtures/pharmcat/pharmcat-docs-example.json, PharmCAT's invented example sample); HLA-A, HLA-B and CYP2D6 become the outside calls step 36 passes on for DEMO-001, worked out by og pgx outside-calls itself; - T1K's genotype table and the CYP2D6 depth check are the invented fixtures of tests/fixtures/pgx; - step 33's table is written by `og qc sample`, and step 25's PRS and ancestry tables by `og prs table`, from made-up somalier, VerifyBamID2 and pgsc_calc files and the score folder `og prs format` writes (written here in its layout: og runs only inside the pipeline's Linux containers); - step 33's mosaic rows come from og qc sex-loss, step 26b's table and painting from og ancestry-fine report, step 35b's table from og paraphase table, step 39's from og couple screen and step 42's from og traits table, each fed made-up inputs; step 46's table and step 35's HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE and LPA_TABLE; og-cn's cargo test port_tests_cn_traits and tests/test_lpa_kiv2.py check that og still writes them), step 49's depth rows with og chip's detection limit, at made-up depths; the tables of steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written here, in the columns their writers use (og for 45 to 53), with the values those writers give for the made-up inputs named in each function. DEMO-001 carries every section, so every card is in the pictures: also the Paraphase card (a PacBio HiFi step, though the invented reads are short) and the couple card, with an invented partner DEMO-002. --sample and --sex write another invented person the same way: the live-site demo (tests/demo/build_live_demo.py) writes DEMO-002, DEMO-001's partner (female), and DEMO-003, their son, so the family page has a trio. A female gets no Y files and her sex checks read two X. A few invented findings make every live-site widget show once in the demo (FINDINGS below): DEMO-002 an FMR1 allele in Stranger's premutation range, a CHIP call and a mosaic loss MoChA reports, DEMO-003 an AnnotSV class 4 deletion. DEMO-003 also gets two results to act on that his page keeps closed (PLANTED below): a ClinVar pathogenic hit in an ACMG secondary-findings gene, and an AnnotSV class 5 deletion over it, at invented positions; his symptom search ranks that gene too. Every sample gets Stranger's thresholds beside ExpansionHunter's counts and step 27's CPIC table, written by og pgx cpic-report from the PharmCAT report. The second form reads DIR/DEMO-001/summary.json (step 24 writes it) and fails when a section is not "ok", so a new report card that this script does not feed yet is caught before a picture shows "Not run". Standard library only. Replaced by `og dev demo-sample` (rust/crates/og-dev/src/demo), which the picture retake and the live-site demo run; this file stays only for the older tests that still import or run it, until they move. """ import argparse import calendar import copy import csv import gzip import json import math import os import random import re import shutil import subprocess import sys import tempfile import time HERE = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.dirname(os.path.dirname(HERE)) sys.path.insert(0, os.path.join(REPO, "bin")) sys.path.insert(0, os.path.join(REPO, "tests")) import collect_summary # noqa: E402 from og_binary import og_binary # noqa: E402 SAMPLE = "DEMO-001" SEX = "male" PGX_FIXTURES = os.path.join(REPO, "tests", "fixtures", "pgx") PHARMCAT_EXAMPLE = os.path.join(REPO, "tests", "fixtures", "pharmcat", "pharmcat-docs-example.json") # GRCh38 primary chromosomes and their lengths. CHROMS = [ ("chr1", 248956422), ("chr2", 242193529), ("chr3", 198295559), ("chr4", 190214555), ("chr5", 181538259), ("chr6", 170805979), ("chr7", 159345973), ("chr8", 145138636), ("chr9", 138394717), ("chr10", 133797422), ("chr11", 135086622), ("chr12", 133275309), ("chr13", 114364328), ("chr14", 107043718), ("chr15", 101991189), ("chr16", 90338345), ("chr17", 83257441), ("chr18", 80373285), ("chr19", 58617616), ("chr20", 64444167), ("chr21", 46709983), ("chr22", 50818468), ("chrX", 156040895), ("chrY", 57227415), ] AUTOSOMES = [c for c in CHROMS if c[0] not in ("chrX", "chrY")] # Variants per base of each chromosome, relative to an autosome (one X, one Y). DENSITY = {"chrX": 0.45, "chrY": 0.04} def own_chroms(): """The chromosomes the invented person has reads and variants on: a female has no chrY.""" return [c for c in CHROMS if SEX == "male" or c[0] != "chrY"] MEAN_DEPTH = 31.3 # When the invented run started: 2026-10-01 09:00 UTC, the time of the demo's # PharmCAT report too (logs/run_status.tsv, the report's "Latest run" line). RUN_STARTED = calendar.timegm((2026, 10, 1, 9, 0, 0, 0, 0, 0)) # The invented findings of the demo's other people (--sample), one of each live-site widget's marks. FINDINGS = {"DEMO-002": {"fmr1", "chip", "mocha"}, "DEMO-003": {"annotsv", "act"}} # The son's results to act on ("act"): a pathogenic ClinVar hit in KCNH2, an ACMG secondary-findings gene that # is on no carrier list, and an AnnotSV class 5 deletion over it. Public gene name, invented positions and # alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose, and that the # family page keeps his places closed with them, the compound-het pairs of tests/fixtures/live/demo/DEMO-FAMILY too. PLANTED = {"gene": "KCNH2", "hit": ("chr7", 150960000, "C", "T"), "sv": ("chr7", 150955000, 150965000)} # Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), "ATXN3": (44, 60), "AR": (34, 38), "FXN": (33, 66)} def has(finding): return finding in FINDINGS.get(SAMPLE, ()) def put(path, text, gz=False): os.makedirs(os.path.dirname(path), exist_ok=True) if gz: with gzip.open(path, "wt", compresslevel=6) as f: f.write(text) else: with open(path, "w") as f: f.write(text) def vcf_header(extra=(), sample=True): lines = ["##fileformat=VCFv4.2"] + [f"##contig=" for c, n in CHROMS] + list(extra) cols = "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO" lines.append(cols + ("\tFORMAT\t" + SAMPLE if sample else "")) return "\n".join(lines) + "\n" def hla_release(): """HLA_DB_RELEASE's default in scripts/lib/common.sh (3.65.0 -> '3.65.0').""" with open(os.path.join(REPO, "scripts", "lib", "common.sh")) as f: for line in f: m = re.match(r'HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-([0-9.]+)\}', line.strip()) if m: return m.group(1) raise SystemExit("HLA_DB_RELEASE is not in scripts/lib/common.sh") def image_tag(var): """The tag of an image line of versions.env (PHARMCAT_IMAGE -> 3.4.0).""" with open(os.path.join(REPO, "versions.env")) as f: for line in f: if line.startswith(var + "="): ref = line.split("=", 1)[1].split("#")[0].strip().strip('"') return ref.split("@")[0].rsplit(":", 1)[-1] raise SystemExit(f"{var} is not in versions.env") # --- step 03: the variant calls ------------------------------------------------------ def variants(d, rng, n): """vcf/S.vcf.gz: N records at random positions, about 94% PASS, 85% SNVs.""" chroms = own_chroms() weights = [length * DENSITY.get(c, 1.0) for c, length in chroms] total_w = sum(weights) path = os.path.join(d, "vcf", f"{SAMPLE}.vcf.gz") os.makedirs(os.path.dirname(path), exist_ok=True) head = vcf_header(['##FILTER=', '##FORMAT=', '##FORMAT=', '##FORMAT=']) bases = "ACGT" with gzip.open(path, "wt", compresslevel=1) as f: f.write(head) for (chrom, length), w in zip(chroms, weights): k = round(n * w / total_w) hap = chrom in ("chrX", "chrY") and SEX == "male" out = [] for pos in sorted(rng.sample(range(10001, length - 10000), k)): ref = bases[rng.randrange(4)] r = rng.random() if r < 0.855: alt = bases[(bases.index(ref) + 1 + rng.randrange(3)) % 4] elif r < 0.93: alt = ref + "".join(bases[rng.randrange(4)] for _ in range(1 + rng.randrange(6))) else: alt, ref = ref, ref + "".join(bases[rng.randrange(4)] for _ in range(1 + rng.randrange(6))) dp = max(3, int(rng.gauss(MEAN_DEPTH / (2 if hap else 1), 6))) if rng.random() < 0.942: gt = "1/1" if hap or rng.random() < 0.38 else "0/1" out.append(f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t{rng.randint(20, 66)}.{rng.randrange(10)}\tPASS\t.\t" f"GT:GQ:DP\t{gt}:{rng.randint(20, 60)}:{dp}\n") else: out.append(f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t{rng.randint(0, 9)}.{rng.randrange(10)}\tRefCall\t.\t" f"GT:GQ:DP\t0/0:{rng.randint(5, 30)}:{dp}\n") f.write("".join(out)) # --- step 06: ClinVar screen ----------------------------------------------------------- def clinvar(d, genome_dir): """Three heterozygous hits in genes with invented names (the picture stops above the hits table, and no real gene is tied to an invented person). DEMO-003 gets a fourth, the planted result to act on (PLANTED): a real gene, needed for the secondary-findings rule, at an invented position.""" hits = [ ("chr2", 47403191, "C", "T", "DEMOA:90001", "Pathogenic", "criteria_provided,_multiple_submitters,_no_conflicts"), ("chr11", 108259012, "G", "A", "DEMOB:90002", "Likely_pathogenic", "criteria_provided,_single_submitter"), ("chr16", 2087914, "CT", "C", "DEMOC:90003", "Pathogenic", "criteria_provided,_multiple_submitters,_no_conflicts"), ] if has("act"): c, p, r, a = PLANTED["hit"] hits.append((c, p, r, a, f"{PLANTED['gene']}:3757", "Pathogenic", "criteria_provided,_multiple_submitters,_no_conflicts")) hits.sort(key=lambda h: ([x[0] for x in CHROMS].index(h[0]), h[1])) body = "".join(f"{c}\t{p}\t{90000 + i}\t{r}\t{a}\t50\tPASS\tGENEINFO={g};CLNSIG={s};CLNREVSTAT={rev}\tGT\t0/1\n" for i, (c, p, r, a, g, s, rev) in enumerate(hits)) put(os.path.join(d, "clinvar", f"{SAMPLE}_clinvar_hits.vcf"), vcf_header() + body) # The ClinVar file itself: only the header line the run manifest reads. put(os.path.join(genome_dir, "clinvar", "clinvar.vcf.gz"), "##fileformat=VCFv4.1\n##fileDate=2026-09-28\n##source=ClinVar\n" "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n", gz=True) # --- pharmacogenomics: steps 07, 08, 21, 32 (36 and 27 run for real) ------------------- def pgx(d, genome_dir, rng): """T1K, pypgx, Cyrius and the depth check, then the PharmCAT report that step 07 would write once step 36 passed their consensus on.""" example = json.load(open(PHARMCAT_EXAMPLE)) genes = example["genes"] def called(gene): g = genes[gene]["sourceDiplotypes"][0] return g["label"], "; ".join(g.get("phenotypes") or []) # T1K (step 08) and the CYP2D6 depth check: the invented fixtures. t1k = os.path.join(d, "hla_t1k", f"{SAMPLE}_hla_genotype.tsv") os.makedirs(os.path.dirname(t1k), exist_ok=True) shutil.copy(os.path.join(PGX_FIXTURES, "t1k_genotype.tsv"), t1k) depth = os.path.join(d, "pypgx", f"{SAMPLE}_cyp2d6_depth_check.tsv") os.makedirs(os.path.dirname(depth), exist_ok=True) shutil.copy(os.path.join(PGX_FIXTURES, "depth_check_ok.tsv"), depth) # The HLA database release the manifest prints: one header line of hla.dat, # naming the release setup.sh installs (HLA_DB_RELEASE in scripts/lib/common.sh). put(os.path.join(genome_dir, "t1k_idx", "hlaidx", "hla.dat"), f"ID HLA00001; standard; DNA; HUM; 3503 BP.\nDT 15/07/2025 (Rel. {hla_release()}, Created)\n") # pypgx (step 32) and Cyrius (step 21) agree with PharmCAT's example calls, # CYP2D6 included, so the three CYP2D6 callers agree. cyp, cyp_phen = called("CYP2D6") rows = [] for gene in ("CYP2B6", "CYP2C19", "CYP2C9", "CYP2D6", "CYP3A5", "NUDT15", "SLCO1B1", "TPMT", "UGT1A1"): dip, phen = called(gene) rows.append(f"{gene}\t{dip}\t{phen}\t{'Normal' if gene == 'CYP2D6' else 'N/A'}\t{'BAM' if gene == 'CYP2D6' else 'VCF'}\n") put(os.path.join(d, "pypgx", f"{SAMPLE}_pypgx_summary.tsv"), "Gene\tDiplotype\tPhenotype\tCNV_call\tSource\n" + "".join(rows)) put(os.path.join(d, "cyrius", f"{SAMPLE}_cyp2d6.tsv"), f"Sample\tGenotype\tFilter\n{SAMPLE}\t{cyp}\tPASS\n") # What step 36 will pass to PharmCAT, from its own code. with tempfile.TemporaryDirectory() as tmp: calls = os.path.join(tmp, "calls.tsv") r = subprocess.run([og_binary(), "pgx", "outside-calls", "--calls", calls, "--consensus", os.path.join(tmp, "consensus.tsv"), "--hla", t1k, "--pypgx", os.path.join(d, "pypgx", f"{SAMPLE}_pypgx_summary.tsv"), "--cyrius", os.path.join(d, "cyrius", f"{SAMPLE}_cyp2d6.tsv"), "--depth-check", depth], stdout=subprocess.DEVNULL) if r.returncode: raise SystemExit(f"og pgx outside-calls failed (exit {r.returncode})") outside = dict(line.rstrip("\n").split("\t") for line in open(calls)) # The PharmCAT report (step 07, run after step 36): PharmCAT's example, # with DEMO-001's outside calls. HLA phenotypes as PharmCAT writes them. report = copy.deepcopy(example) report.pop("_fixture_note", None) report["title"] = SAMPLE report["timestamp"] = "2026-10-01T09:00:00.000Z" report["pharmcatVersion"] = image_tag("PHARMCAT_IMAGE") hla_tests = {"HLA-A": ["*31:01"], "HLA-B": ["*15:02", "*57:01", "*58:01"]} changed = {} for gene, dip in outside.items(): alleles = dip.split("/") if gene in hla_tests: phen = [f"{t} {'positive' if t in alleles else 'negative'}" for t in hla_tests[gene]] else: phen = [cyp_phen] if dip == cyp else ["Indeterminate"] if dip != called(gene)[0]: changed[gene] = dip report["genes"][gene]["callSource"] = "OUTSIDE" report["genes"][gene]["sourceDiplotypes"] = [{ "gene": gene, "label": dip, "phenotypes": phen, "activityScore": None, "outsidePhenotype": False, "allele1": {"gene": gene, "name": alleles[0], "function": None}, "allele2": {"gene": gene, "name": alleles[1], "function": None}}] # MT-RNR1 is an outside call in PharmCAT's example; this pipeline passes # none for it, so it is read from the VCF here, as the reference allele. mt = report["genes"]["MT-RNR1"] mt["callSource"] = "MATCHER" mt["sourceDiplotypes"] = [{"gene": "MT-RNR1", "label": "Reference", "phenotypes": ["normal risk of aminoglycoside-induced hearing loss"], "activityScore": None, "outsidePhenotype": False, "allele1": {"gene": "MT-RNR1", "name": "Reference", "function": None}, "allele2": None}] changed["MT-RNR1"] = "Reference" prune_annotations(report, changed) put(os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), json.dumps(report, indent=1) + "\n") # Step 27's tables from that report, by the parser step 27 runs (og pgx). r = subprocess.run([og_binary(), "pgx", "cpic-report", "--sample", SAMPLE, "--report", os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), "--outdir", os.path.join(d, "cpic")], stdout=subprocess.DEVNULL) if r.returncode: raise SystemExit(f"og pgx cpic-report failed (exit {r.returncode})") return outside def prune_annotations(report, changed): """Drop the example's drug annotations written for a diplotype of a gene whose call changed: PharmCAT lists only those that fit the sample.""" for src, by_drug in list(report.get("drugs", {}).items()): for name, rep in list(by_drug.items()): for gl in rep.get("guidelines") or []: keep = [] for ann in gl.get("annotations") or []: labels = {(dp.get("gene"), dp.get("label")) for gt in ann.get("genotypes") or [] for dp in (gt or {}).get("diplotypes") or [] if isinstance(dp, dict)} stale = any(g in changed and lab != changed[g] for g, lab in labels) stale = stale or any(g in changed and not any(x[0] == g for x in labels) for g in (ann.get("phenotypes") or {})) if not stale: keep.append(ann) gl["annotations"] = keep rep["guidelines"] = [gl for gl in rep.get("guidelines") or [] if gl.get("annotations")] if not rep["guidelines"]: del by_drug[name] if not by_drug: del report["drugs"][src] # --- step 25: polygenic scores through pgsc_calc's files ------------------------------- def prs(d, genome_dir, rng): """The harmonised files (their headers feed the run manifest) and the score folder og prs format would write from them, without the allele list, which nothing here reads: every row is autosomal, so it is written, an A/T or C/G pair is counted as strand-ambiguous, and no chrX row is left out.""" score_list = os.path.join(REPO, "assets", "pgs_scores.tsv") with open(score_list, encoding="utf-8") as f: listed = [r for r in csv.DictReader((line for line in f if not line.startswith("#")), delimiter="\t") if (r.get("pgs_id") or "").strip()] labels = {r["pgs_id"].strip(): (r.get("trait_reported") or "").strip() for r in listed} applies = {r["pgs_id"].strip(): (r.get("applies_to") or "").strip().lower() for r in listed} # The evaluations as og prs format copies them: the three ;-lists, each entry stripped. evals = {r["pgs_id"].strip(): "\t".join(";".join(x.strip() for x in (r.get(k) or "").split(";")) if (r.get(k) or "").strip() else "" for k in ("ppm_id", "ancestry", "effect")) for r in listed if any((r.get(k) or "").strip() for k in ("ppm_id", "ancestry", "effect"))} harmonised = os.path.join(genome_dir, "prs_scores") work = os.path.join(d, "prs", "pgsc_calc") scores = os.path.join(work, "scores") ambiguous = {} for pid, trait in labels.items(): n = rng.randint(250, 4000) rows, written = [], [] for _ in range(n): c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] ea, oa = rng.sample("ACGT", 2) pos, w, hm_pos = rng.randint(1, length), f"{rng.gauss(0, 0.05):.5f}", rng.randint(1, length) rows.append(f"{c[3:]}\t{pos}\t{ea}\t{oa}\t{w}\t{c[3:]}\t{hm_pos}\n") written.append(f"{c[3:]}\t{hm_pos}\t{ea}\t{oa}\t{w}\n") ambiguous[pid] = sum(1 for r in written if r.split("\t")[2] + r.split("\t")[3] in ("AT", "TA", "CG", "GC")) put(os.path.join(harmonised, f"{pid}.txt.gz"), f"#format_version=2.0\n#pgs_id={pid}\n#pgs_name={pid}\n#trait_reported={trait}\n#variants_number={n}\n" "#HmPOS_build=GRCh38\nchr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\thm_chr\thm_pos\n" + "".join(rows), gz=True) put(os.path.join(scores, f"{pid}.txt.gz"), f"#pgs_id={pid}\n#pgs_name={pid}\n#trait_reported={(trait or pid).replace('=', '-')}\n#genome_build=GRCh38\n" "chr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\n" + "".join(written), gz=True) ids = sorted(labels) put(os.path.join(scores, "chrx_rows.tsv"), "pgs_id\tchrx_rows\tchrx\n" + "".join(f"{p}\t0\tleft out\n" for p in ids)) put(os.path.join(scores, "score_notes.tsv"), "pgs_id\tapplies_to\tambiguous_rows\tambiguous\n" + "".join(f"{p}\t{applies.get(p, '')}\t{ambiguous[p]}\tdropped\n" for p in ids)) put(os.path.join(scores, "score_evaluations.tsv"), "pgs_id\tppm_id\tancestry\teffect\n" + "".join(f"{p}\t{evals[p]}\n" for p in ids if p in evals)) # pgsc_calc's output for sampleset "sample" (step 25's name) with the 1000 Genomes panel. res = os.path.join(work, "results", "sample") summ, score = ["dataset,accession,ambiguous,is_multiallelic,match_flipped,duplicate_best_match," "duplicate_ID,match_IDs,match_status,count,score_pass,match_rate"], [] for pid in sorted(labels): with gzip.open(os.path.join(scores, f"{pid}.txt.gz"), "rt") as f: total = sum(1 for line in f if not line.startswith("#")) - 1 m = round(total * rng.uniform(0.86, 0.99)) summ.append(f"sample,{pid},false,false,false,false,false,NA,matched,{m},true,{m / total:.2f}") summ.append(f"sample,{pid},false,false,false,false,false,NA,unmatched,{total - m},true,{m / total:.2f}") score.append(f"sample\t{SAMPLE}\t{SAMPLE}\t{pid}\t{rng.gauss(0, 0.4):.4f}\t{rng.gauss(0, 1):.3f}\t" f"{rng.uniform(18, 82):.2f}") put(os.path.join(res, "match", "sample_summary.csv"), "\n".join(summ) + "\n") put(os.path.join(res, "score", "sample_pgs.txt.gz"), "sampleset\tFID\tIID\tPGS\tSUM\tZ_MostSimilarPop\tpercentile_MostSimilarPop\n" + "\n".join(score) + "\n", gz=True) pcs = "\t".join(f"{rng.gauss(0, 0.01):.4f}" for _ in range(10)) put(os.path.join(res, "score", "sample_popsimilarity.txt.gz"), "sampleset\tFID\tIID\t" + "\t".join(f"PC{i}" for i in range(1, 11)) + "\tRF_P_AFR\tRF_P_AMR\tRF_P_EAS\tRF_P_EUR\tRF_P_SAS\tMostSimilarPop\tMostSimilarPop_LowConfidence\tREFERENCE\n" f"sample\t{SAMPLE}\t{SAMPLE}\t{pcs}\t0.00\t0.02\t0.00\t0.97\t0.01\tEUR\tFalse\tFalse\n", gz=True) os.makedirs(os.path.join(d, "ancestry"), exist_ok=True) r = subprocess.run([og_binary(), "prs", "table", "--sample", SAMPLE, "--results", os.path.join(work, "results"), "--sampleset", "sample", "--scores", scores, "--input-kind", "gvcf", "--panel", image_tag_panel(), "--ancestry-out", os.path.join(d, "ancestry", f"{SAMPLE}_ancestry.tsv"), "--out", os.path.join(d, "prs", f"{SAMPLE}_prs_summary.tsv")], capture_output=True, text=True) if r.returncode: raise SystemExit(f"og prs table failed: {r.stderr.strip()}") def image_tag_panel(): with open(os.path.join(REPO, "versions.env")) as f: for line in f: if line.startswith("PGSC_PANEL="): return line.split("=", 1)[1].split("#")[0].strip().strip('"') return "pgsc_1000G_v1" # --- steps 16, 16b, 33, 01b and samtools: QC ------------------------------------------ def coverage(d, rng): """mosdepth (step 16b) and indexcov (step 16).""" m = os.path.join(d, "mosdepth") summary = ["chrom\tlength\tbases\tmean\tmin\tmax"] dist = [] tot_len = tot_bases = 0 def cumulative(mean, sd, gap): # Share of bases with at least each depth: a normal curve, plus the # gaps of the reference (no reads) at depth 0. out = [] for dep in range(0, int(mean * 2.6) + 1): p = (1 - gap) * 0.5 * math.erfc((dep - 0.5 - mean) / (sd * math.sqrt(2))) if dep else 1.0 if p < 0.005 and dep: break out.append((dep, p)) return out curves = {} for chrom, length in own_chroms(): half = chrom in ("chrX", "chrY") and SEX == "male" mean = MEAN_DEPTH / (2 if half else 1) * rng.uniform(0.97, 1.03) gap = rng.uniform(0.01, 0.03) if chrom != "chrY" else 0.55 bases = int(length * mean * (1 - gap)) summary.append(f"{chrom}\t{length}\t{bases}\t{bases / length:.2f}\t0\t{int(mean * 9)}") tot_len += length tot_bases += bases curves[chrom] = cumulative(mean, mean * 0.24, gap) summary.append(f"total\t{tot_len}\t{tot_bases}\t{tot_bases / tot_len:.2f}\t0\t{int(MEAN_DEPTH * 9)}") for chrom, _ in own_chroms(): dist += [f"{chrom}\t{dep}\t{p:.2f}" for dep, p in curves[chrom]] dist += [f"total\t{dep}\t{p:.2f}" for dep, p in (cumulative(MEAN_DEPTH, MEAN_DEPTH * 0.25, 0.02))] put(os.path.join(m, f"{SAMPLE}.mosdepth.summary.txt"), "\n".join(summary) + "\n") put(os.path.join(m, f"{SAMPLE}.mosdepth.global.dist.txt"), "\n".join(dist) + "\n") i = os.path.join(d, "indexcov") put(os.path.join(i, "indexcov-indexcov.ped"), "#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\tCNchrX\tCNchrY\tbins.out\tbins.lo\tbins.hi\t" "bins.in\tslope\tp.out\tPC1\tPC2\tPC3\tPC4\tPC5\n" f"unknown\t{SAMPLE}\t-9\t-9\t{1 if SEX == 'male' else 2}\t-9\t{'1.01' if SEX == 'male' else '1.99'}\t" f"{'0.98' if SEX == 'male' else '0.01'}\t312\t205\t107\t174820\t0.993\t0.0018\t" "0\t0\t0\t0\t0\n") roc = ["#chrom\tcov\t" + SAMPLE] for chrom, _ in own_chroms(): half = chrom in ("chrX", "chrY") and SEX == "male" centre = 0.5 if half else 1.0 for k in range(0, 41): cov = k * 0.05 frac = 0.5 * math.erfc((cov - centre) / (0.12 * math.sqrt(2))) roc.append(f"{chrom}\t{cov:.2f}\t{min(1.0, frac):.3f}") put(os.path.join(i, "indexcov-indexcov.roc"), "\n".join(roc) + "\n") def sample_qc(d): """somalier and VerifyBamID2 files, then step 33's own table writer. sex is the sex somalier infers from the reads. original_pedigree_sex stays -9 (unknown), as in a real run: step 33 gives somalier no pedigree file, so MultiQC's somalier "Sex" column (that pedigree sex) reads -9.""" q = os.path.join(d, "qc") sex = 1 if SEX == "male" else 2 put(os.path.join(q, "somalier", f"{SAMPLE}.samples.tsv"), "#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\toriginal_pedigree_sex\tgt_depth_mean\t" "gt_depth_sd\tdepth_mean\tdepth_sd\tab_mean\tab_std\tn_hom_ref\tn_het\tn_hom_alt\tn_unknown\tp_middling_ab\t" "X_depth_mean\tX_n\tX_hom_ref\tX_het\tX_hom_alt\tY_depth_mean\tY_n\n" f"{SAMPLE}\t{SAMPLE}\t-9\t-9\t{sex}\t-9\t-9\t31.4\t7.2\t31.4\t7.2\t0.41\t0.08\t9817\t6904\t4383\t196\t0.01\t" + ("15.6\t612\t352\t4\t256\t14.9\t41\n" if SEX == "male" else "31.2\t612\t300\t210\t102\t0.1\t0\n")) put(os.path.join(q, "somalier", f"{SAMPLE}.pairs.tsv"), "#sample_a\tsample_b\trelatedness\tibs0\tibs2\thom_concordance\thets_a\thets_b\thets_ab\tshared_hets\t" "hom_alts_a\thom_alts_b\tshared_hom_alts\tn\tx_ibs0\tx_ibs2\texpected_relatedness\n") put(os.path.join(q, "verifybamid2", f"{SAMPLE}.selfSM"), "#SEQ_ID\tRG\tCHIP_ID\t#SNPS\t#READS\tAVG_DP\tFREEMIX\tFREELK1\tFREELK0\tFREE_RH\tFREE_RA\tCHIPMIX\tCHIPLK1\t" "CHIPLK0\tCHIP_RH\tCHIP_RA\tDPREF\tRDPHET\tRDPALT\n" f"{SAMPLE}\tNA\tNA\t99846\t3087112\t30.92\t0.00412\t-1418823.2\t-1419102.7\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\n") r = subprocess.run([og_binary(), "qc", "sample", "--sample", SAMPLE, "--somalier-samples", os.path.join(q, "somalier", f"{SAMPLE}.samples.tsv"), "--somalier-pairs", os.path.join(q, "somalier", f"{SAMPLE}.pairs.tsv"), "--selfsm", os.path.join(q, "verifybamid2", f"{SAMPLE}.selfSM"), "--declared-sex", SEX, "--marker-check", "passed", "--out", os.path.join(q, f"{SAMPLE}_sample_qc.tsv")], capture_output=True, text=True) if r.returncode: raise SystemExit(f"og qc sample failed: {r.stderr.strip()}") def reads(d, rng): """fastp's JSON (step 01b) and samtools flagstat (step 28 writes it from the BAM).""" pairs = 412_906_118 before = pairs * 2 passed = int(before * 0.9891) low_q, too_short, n_reads = int(before * 0.0081), int(before * 0.0026), before - passed - int(before * 0.0081) - int(before * 0.0026) length = 151 def curves(after): q = [round(min(37.4, 31.5 + 6 * (1 - math.exp(-i / 4))) - (i / length) ** 3 * (2.2 if after else 3.5) + rng.uniform(-0.15, 0.15), 2) for i in range(length)] base = {b: [round(v + rng.uniform(-0.004, 0.004), 4) for v in [p] * length] for b, p in (("A", 0.295), ("T", 0.296), ("C", 0.204), ("G", 0.205))} content = dict(base, N=[0.0002] * length, GC=[round(base["C"][i] + base["G"][i], 4) for i in range(length)]) return {"total_reads": (passed if after else before) // 2, "total_bases": (passed if after else before) // 2 * length, "q20_bases": int((passed if after else before) // 2 * length * 0.968), "q30_bases": int((passed if after else before) // 2 * length * 0.921), "total_cycles": length, "quality_curves": dict({b: q for b in "ATCG"}, mean=q), "content_curves": content} def stats(n, gc): return {"total_reads": n, "total_bases": n * length, "q20_bases": int(n * length * 0.968), "q30_bases": int(n * length * 0.921), "q20_rate": 0.968, "q30_rate": 0.921, "read1_mean_length": length, "read2_mean_length": length, "gc_content": gc} hist = [0] * 1000 for i in range(1000): hist[i] = int(4.2e6 * math.exp(-((i - 382) / 118) ** 2 / 2)) fastp = { "summary": {"fastp_version": "1.0.1", "sequencing": "paired end (151 cycles + 151 cycles)", "before_filtering": stats(before, 0.4105), "after_filtering": stats(passed, 0.4098)}, "filtering_result": {"passed_filter_reads": passed, "low_quality_reads": low_q, "too_many_N_reads": n_reads, "too_short_reads": too_short, "too_long_reads": 0}, "duplication": {"rate": 0.0712}, "insert_size": {"peak": 382, "unknown": 5102117, "histogram": hist}, "adapter_cutting": {"adapter_trimmed_reads": int(before * 0.074), "adapter_trimmed_bases": int(before * 1.9), "read1_adapter_sequence": "AGATCGGAAGAGCACACGTCTGAACTCCAGTCA", "read2_adapter_sequence": "AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT"}, "read1_before_filtering": curves(False), "read2_before_filtering": curves(False), "read1_after_filtering": curves(True), "read2_after_filtering": curves(True), "command": f"fastp -i /genome/{SAMPLE}/fastq/{SAMPLE}_R1.fastq.gz -I /genome/{SAMPLE}/fastq/{SAMPLE}_R2.fastq.gz " f"-o /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_R1.fastq.gz -O /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_R2.fastq.gz " f"-j /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_fastp.json -h /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_fastp.html", } put(os.path.join(d, "fastq_trimmed", f"{SAMPLE}_fastp.json"), json.dumps(fastp, indent=1) + "\n") primary = passed supp = int(primary * 0.0034) total = primary + supp mapped = int(primary * 0.9962) + supp dups = int(primary * 0.0712) paired = primary proper = int(paired * 0.981) single = int(paired * 0.0019) both = int(paired * 0.9943) diff = int(paired * 0.0061) flag = [ f"{total} + 0 in total (QC-passed reads + QC-failed reads)", f"{primary} + 0 primary", "0 + 0 secondary", f"{supp} + 0 supplementary", f"{dups} + 0 duplicates", f"{dups} + 0 primary duplicates", f"{mapped} + 0 mapped ({100 * mapped / total:.2f}% : N/A)", f"{mapped - supp} + 0 primary mapped ({100 * (mapped - supp) / primary:.2f}% : N/A)", f"{paired} + 0 paired in sequencing", f"{paired // 2} + 0 read1", f"{paired // 2} + 0 read2", f"{proper} + 0 properly paired ({100 * proper / paired:.2f}% : N/A)", f"{both} + 0 with itself and mate mapped", f"{single} + 0 singletons ({100 * single / paired:.2f}% : N/A)", f"{diff} + 0 with mate mapped to a different chr", f"{int(diff * 0.62)} + 0 with mate mapped to a different chr (mapQ>=5)", ] put(os.path.join(d, "aligned", f"{SAMPLE}_flagstat.txt"), "\n".join(flag) + "\n") # --- structural variants: steps 04, 18, 19, 22 ------------------------------------------ def sv_records(rng, n, caller, pass_rate): out = [] for i in range(n): c, length = CHROMS[rng.randrange(len(CHROMS) - 1)] t = rng.choices(["DEL", "DUP", "INV", "INS"], weights=[62, 14, 6, 18])[0] pos = rng.randint(20000, length - 2_000_000) size = int(math.exp(rng.uniform(math.log(60), math.log(400000)))) end = pos + (1 if t == "INS" else size) svlen = -size if t == "DEL" else size filt = "PASS" if rng.random() < pass_rate else rng.choice(["MinQUAL", "MinGQ", "LowQual"]) gt = "1/1" if rng.random() < 0.2 else "0/1" out.append((c, pos, f"{caller}{t}:{i}", t, end, svlen, filt, gt, rng.randint(20, 999))) out.sort(key=lambda r: ([x[0] for x in CHROMS].index(r[0]), r[1])) return out def structural(d, rng): info = ['##INFO=', '##INFO=', '##INFO='] def write(path, recs, extra=()): body = "".join(f"{c}\t{p}\t{i}\tN\t<{t}>\t{q}\t{f}\tEND={e};SVTYPE={t};SVLEN={l}{x}\tGT\t{g}\n" for (c, p, i, t, e, l, f, g, q), x in zip(recs, extra or [""] * len(recs))) put(path, vcf_header(info + ['##FORMAT=']) + body, gz=True) write(os.path.join(d, "manta", "results", "variants", "diploidSV.vcf.gz"), sv_records(rng, rng.randint(7600, 8000), "Manta", 0.82)) write(os.path.join(d, "delly", f"{SAMPLE}_sv.vcf.gz"), sv_records(rng, rng.randint(5900, 6300), "Delly", 0.86)) # SURVIVOR's merge of Manta, Delly and CNVpytor (step 22): SUPP and SUPP_VEC. merged = sv_records(rng, rng.randint(2700, 3100), "SURVIVOR", 1.0) vecs = [rng.choices(["110", "101", "011", "111"], weights=[70, 9, 6, 15])[0] for _ in merged] write(os.path.join(d, "sv_merged", f"{SAMPLE}_sv_consensus.vcf.gz"), merged, [f";SUPP={v.count('1')};SUPP_VEC={v}" for v in vecs]) lines = [] for _ in range(rng.randint(1100, 1250)): c, length = CHROMS[rng.randrange(len(CHROMS) - 1)] t = "deletion" if rng.random() < 0.83 else "duplication" size = rng.randrange(1000, 400000, 100) start = rng.randrange(10001, length - size, 100) cn = rng.uniform(0.02, 0.6) if t == "deletion" else rng.uniform(1.3, 2.1) p = 10 ** rng.uniform(-40, -0.5) lines.append(f"{t}\t{c}:{start}-{start + size - 1}\t{size}\t{cn:.4f}\t{p:.4e}\t{p * 3:.4e}\t{min(1, p * 5):.4e}\t" f"{min(1, p * 9):.4e}\t{rng.uniform(0, 0.5):.4f}\t{rng.uniform(0, 0.1):.4f}\t{rng.randint(0, 9)}") put(os.path.join(d, "cnvpytor", f"{SAMPLE}_cnvs.txt"), "\n".join(lines) + "\n") # Step 05's class 4 and 5 SVs, checked (the columns og sv postfilter # writes): none, as in most genomes. cols = ["AnnotSV_ID", "ID", "SV_chrom", "SV_start", "SV_end", "SV_length", "SV_type", "Gene_name", "ACMG_class", "AnnotSV_ranking_score", "Caller", "Exon_edge", "Exon_edge_detail", "Population_AF", "Population_source", "Common", "AnnotSV_class", "Depth_support", "Rule", "Rule_detail"] rows = [] if has("annotsv"): # One class 4 deletion over an invented gene, from Manta, kept by step 05's rules. rows.append({"AnnotSV_ID": "chr16_2084000_2096000_DEL_1", "ID": "MantaDEL:1", "SV_chrom": "chr16", "SV_start": "2084000", "SV_end": "2096000", "SV_length": "-12000", "SV_type": "DEL", "Gene_name": "G0533", "ACMG_class": "4", "AnnotSV_ranking_score": "0.92", "Caller": "Manta", "Exon_edge": "no", "Exon_edge_detail": ".", "Population_AF": "", "Population_source": "", "Common": "no", "AnnotSV_class": "4", "Depth_support": ".", "Rule": "", "Rule_detail": ""}) if has("act"): # The class 5 deletion over the planted ClinVar hit: a result to act on. c, a, b = PLANTED["sv"] rows.append({"AnnotSV_ID": f"{c}_{a}_{b}_DEL_1", "ID": "MantaDEL:2", "SV_chrom": c, "SV_start": str(a), "SV_end": str(b), "SV_length": str(a - b), "SV_type": "DEL", "Gene_name": PLANTED["gene"], "ACMG_class": "5", "AnnotSV_ranking_score": "0.99", "Caller": "Manta", "Exon_edge": "no", "Exon_edge_detail": ".", "Population_AF": "", "Population_source": "", "Common": "no", "AnnotSV_class": "5", "Depth_support": ".", "Rule": "", "Rule_detail": ""}) put(os.path.join(d, "annotsv", f"{SAMPLE}_sv_class4_5_checked.tsv"), "\t".join(cols) + "\n" + "".join("\t".join(r[k] for k in cols) + "\n" for r in rows)) # --- repeats, telomeres, ROH, mitochondria, Y: steps 09, 10, 11, 12, 20, 37 -------------- # ExpansionHunter's key loci with the normal range each repeat count is drawn from. EH_RANGES = [("HTT", "chr4", 3074877, "CAG", 10, 26), ("FMR1", "chrX", 147912051, "CGG", 20, 40), ("C9ORF72", "chr9", 27573529, "GGCCCC", 2, 10), ("ATXN1", "chr6", 16327636, "TGC", 25, 35), ("DMPK", "chr19", 45770205, "CAG", 5, 30), ("ATXN3", "chr14", 92071011, "GCT", 14, 30), ("AR", "chrX", 67545317, "GCA", 17, 26), ("FXN", "chr9", 69037287, "GAA", 6, 30)] def repeats_and_more(d, rng): eh, stranger = [], [] for rep, c, pos, unit, lo, hi in EH_RANGES: hap = c == "chrX" and SEX == "male" a = sorted(rng.randint(lo, hi) for _ in range(1 if hap else 2)) if rep == "FMR1" and has("fmr1") and not hap: a = [a[0], 62] # an allele in Stranger's premutation range (above 54, under 200) ref = (lo + hi) // 2 gt = "1" if hap else ("1/2" if a[0] != a[1] else "1/1") cn = "/".join(map(str, a)) alts = ",".join(f"" for x in dict.fromkeys(a)) info = f"END={pos + ref * len(unit)};REF={ref};RL={ref * len(unit)};RU={unit};VARID={rep};REPID={rep}" lc = f"{rng.uniform(28, 34):.6f}" eh.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info}\tGT:SO:REPCN:LC\t{gt}:SPANNING:{cn}:{lc}\n") # Step 9b's Stranger copy: the same record with the catalog's thresholds, the worst status of # the alleles, and ExpansionHunter's 95% interval of each (REPCI), a few repeats wide past 50. nmax, pmin = STR_THRESHOLDS[rep] worst = ("full_mutation" if any(x >= pmin for x in a) else "pre_mutation" if any(x > nmax for x in a) else "normal") ci = "/".join(f"{x}-{x}" if x < 50 else f"{x - 4}-{x + 5}" for x in a) stranger.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info};STR_STATUS={worst};STR_NORMAL_MAX={nmax};" f"STR_PATHOLOGIC_MIN={pmin}\tGT:SO:REPCN:REPCI:LC\t{gt}:SPANNING:{cn}:{ci}:{lc}\n") head = vcf_header(['##ALT=']) put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh.vcf"), head + "".join(eh)) put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh_stranger.vcf"), head + "".join(stranger)) put(os.path.join(d, "telomere", SAMPLE, f"{SAMPLE}_summary.tsv"), "PID\tsample\ttotal_reads\tread_lengths\trepeat_threshold_set\trepeat_threshold_used\tintratelomeric_reads\t" "gc_bins_for_correction\ttotal_reads_with_tel_gc\ttel_content\n" f"{SAMPLE}\ttumor\t825812236\t151\tn=7 per 100 bp\t10\t61873\t48-52\t185204533\t{rng.uniform(260, 480):.2f}\n") segs, total = [], 0 while total < 44e6: c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] size = int(rng.uniform(0.5e6, 3.9e6)) start = rng.randint(1_000_000, length - size - 1_000_000) segs.append((AUTOSOMES.index((c, length)), c, start, start + size, size)) total += size segs.sort() put(os.path.join(d, "vcf", f"{SAMPLE}_roh.txt"), "# RG, Regions [2]Sample\t[3]Chromosome\t[4]Start\t[5]End\t[6]Length (bp)\t[7]Number of markers\t[8]Quality\n" + "".join(f"RG\t{SAMPLE}\t{c}\t{s}\t{e}\t{n}\t{n // 1400}\t{rng.uniform(40, 90):.1f}\n" for _, c, s, e, n in segs)) mito = [] het = {11, 23} for i, pos in enumerate(sorted(rng.sample(range(60, 16500), 41))): ref = "ACGT"[rng.randrange(4)] alt = "ACGT"[("ACGT".index(ref) + 2) % 4] if i in het: af = rng.uniform(0.11, 0.38) else: af = rng.uniform(0.985, 1.0) if i < 37 else rng.uniform(0.01, 0.03) filt = "PASS" if i < 37 else "weak_evidence" mito.append(f"chrM\t{pos}\t.\t{ref}\t{alt}\t.\t{filt}\t.\tGT:AF:DP\t0/1:{af:.3f}:{rng.randint(2400, 3300)}\n") put(os.path.join(d, "mito", f"{SAMPLE}_chrM_filtered.vcf.gz"), "##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + SAMPLE + "\n" + "".join(mito), gz=True) put(os.path.join(d, "mito", f"{SAMPLE}_haplogroup.txt"), '"SampleID"\t"Haplogroup"\t"Rank"\t"Quality"\t"Range"\n' f'"{SAMPLE}"\t"H1c3"\t"1"\t"0.9312"\t"1-16569;"\n') # Read once wave 11's report has the haplocheck line and the Y card. put(os.path.join(d, "mito", f"{SAMPLE}_haplocheck.txt"), '"Sample"\t"Contamination Status"\t"Contamination Level"\t"Distance"\t"Sample Coverage"\n' f'"{SAMPLE}"\t"NO"\t"ND"\t"0"\t"2861"\n') if SEX != "male": return # no Y chromosome: step 37 has nothing to place put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_haplogroup.txt"), "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" f"{SAMPLE}_sorted\tI-M253\tM253\t4021876\t1187\t0.982\t1.0\t0.991\t0.982\n") # Step 37's placement on the current tree (og y tree-place's columns), read # by the report's y_tree section; the same I-M253 as above, five levels down. # As the real writer does: '.' for an empty SNP list, and its default depth of 10. put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_tree.txt"), "Sample_name\tHg\tLevel\tPath_derived\tPath_ancestral\tPath_no_call\tAncestral_in_terminal\t" "SNPs_called\tMin_depth\tTree\tPositions\tOutvoted_ancestral\n" f"{SAMPLE}\tI-M253\t5\t41\t0\t3\t.\t1187\t10\tYFull YTree v14.06.0\tYleaf hg38 position tables\t.\n") put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_tree_path.tsv"), "Level\tBranch\tSNPs\tOn_tables\tDerived\tAncestral\tNo_call\tAncestral_SNPs\n" "1\tCT\t12\t12\t11\t0\t1\t.\n" "2\tF\t9\t9\t9\t0\t0\t.\n" "3\tIJ\t8\t8\t7\t0\t1\t.\n" "4\tI\t7\t7\t7\t0\t0\t.\n" "5\tI-M253\t8\t8\t7\t0\t1\t.\n") # --- steps 17, 23, 31 ---------------------------------------------------------------- def clinical(d, rng): def gene(): return f"G{rng.randint(1, 900):04d}" impacts = ["HIGH"] * 41 + ["MODERATE"] * 233 + ["LOW"] * 44 rows = [] for imp in impacts: c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] rows.append(f"{c}\t{rng.randint(1, length)}\tC\tT\t0/1\t{imp}\t{gene()}\tmissense_variant\t0.0004\t24.1\t0.41\t.\n") put(os.path.join(d, "clinical", f"{SAMPLE}_clinical_summary.tsv"), "CHROM\tPOS\tREF\tALT\tGT\tIMPACT\tGENE\tConsequence\tMAX_AF\tCADD_PHRED\tREVEL\tAM_CLASS\n" + "".join(rows)) sl = [f"chr{rng.randint(1, 22)}\t{rng.randint(1, 90_000_000)}\tA\tG\t{rng.choice(['HIGH', 'MODERATE'])}\t{gene()}\n" for _ in range(57)] put(os.path.join(d, "slivar", f"{SAMPLE}_slivar_summary.tsv"), "CHROM\tPOS\tREF\tALT\tIMPACT\tGENE\n" + "".join(sl)) ch = [] for _ in range(3): g = gene() ch += [f"chr{rng.randint(1, 22)}\t{rng.randint(1, 90_000_000)}\tA\tG\tMODERATE\t{g}\n" for _ in range(2)] put(os.path.join(d, "slivar", f"{SAMPLE}_compound_hets.tsv"), "CHROM\tPOS\tREF\tALT\tIMPACT\tGENE\n" + "".join(ch)) counts = {"Benign": 289, "Likely_Benign": 112, "VUS": 38} cls = "".join(f"{SAMPLE}\tchr{rng.randint(1, 22)}:g.{rng.randint(1, 90_000_000)}C>T\t{k}\t{k}\n" for k, n in counts.items() for _ in range(n)) put(os.path.join(d, "cpsr", f"{SAMPLE}.cpsr.grch38.classification.tsv.gz"), "SAMPLE_ID\tGENOMIC_CHANGE\tCPSR_CLASSIFICATION\tCLASSIFICATION\n" + cls, gz=True) put(os.path.join(d, "cpsr", f"{SAMPLE}.cpsr.grch38.html"), f"CPSR {SAMPLE}Invented sample.\n") # --- steps 02, 33 (mosaic), 26b, 35b, 39, 40, 41 and 42 --------------------------------- # The sections added since wave 18. Each table is written by the writer its # step uses (bin/*.py) from made-up inputs where that writer is a function, # else in the writer's own columns. def read_count(d): """qc/S_read_count.tsv as read_count_check (scripts/lib/common.sh) writes it: the BAM holds every read fastp kept, so the check is ok.""" with open(os.path.join(d, "fastq_trimmed", f"{SAMPLE}_fastp.json")) as f: kept = json.load(f)["filtering_result"]["passed_filter_reads"] r1 = r2 = kept // 2 src = (f"fastq_trimmed/{SAMPLE}_R1.fastq.gz {r1} records + fastq_trimmed/{SAMPLE}_R2.fastq.gz {r2} records") put(os.path.join(d, "qc", f"{SAMPLE}_read_count.tsv"), f"key\tvalue\ninput_reads\t{r1 + r2}\ninput_source\t{src}\nprimary_records\t{r1 + r2}\n" "missing_reads\t0\nread_count_check\tok\n") # Genes the demo leaves below step 16b's share: four that another step owns and # four with no other reading, so both lines of the card have names in them. DEMO_NOT_READ = {"SMN1", "SMN2", "NCF1", "PMS2", "NBPF1", "ANKRD36C", "DUX4", "MUC12"} def callability(d, rng): """Step 16b's gene table (mosdepth --thresholds 10 over the genes of assets/callability_genes.tsv: chrom, start, end, gene, bases at 10x), written from the repository's catalog, so it always matches it. The ALT-only genes get no row, as in the real step.""" rows = ["#chrom\tstart\tend\tregion\t10X"] with open(os.path.join(REPO, "assets", "callability_genes.tsv"), encoding="utf-8") as f: catalog = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) for r in catalog: if r["start"] == ".": continue start, end = int(r["start"]), int(r["end"]) share = rng.uniform(0.35, 0.8) if r["gene"] in DEMO_NOT_READ else rng.uniform(0.93, 1.0) rows.append(f"{r['chrom']}\t{start}\t{end}\t{r['gene']}\t{int((end - start) * share)}") put(os.path.join(d, "mosdepth", f"{SAMPLE}_callability.bed.gz"), "\n".join(rows) + "\n", gz=True) # The first ten single-copy pieces of chrY and chrX that og qc sex-loss reads # (rust/crates/og-qc/src/sex_loss.rs, SINGLE_COPY_Y and SINGLE_COPY_X). SINGLE_COPY_FIRST = { "Y": [(2781479, 3050044), (6748713, 7574481), (11037032, 11592902), (11674123, 13984473), (14058733, 15875093), (15905214, 16159795), (16425965, 17456266), (18870334, 20207793), (20257793, 21305953), (26415435, 26673214)], "X": [(2781479, 37099262), (37285837, 48343310), (48434604, 49157254), (49528394, 50228964), (50278964, 52032464), (52223402, 52489200), (52520537, 55437684), (55547739, 58100000), (61000000, 62412542), (62462542, 63185573)], } def mosaic(d, rng): """mosdepth's 500 bp windows (step 16b's regions.bed.gz), cut down to three 200 kb stretches of each autosome and the single-copy part of chrY, then og qc sex-loss itself adds its rows to step 33's table. One Y in every cell: the ratio sits near 0.5 and no loss is detected.""" path = os.path.join(d, "mosdepth", f"{SAMPLE}.regions.bed.gz") lines = [] for chrom, length in AUTOSOMES: mean = MEAN_DEPTH * rng.uniform(0.98, 1.02) for k in range(3): start = (length // 4) * (k + 1) // 500 * 500 lines += [f"{chrom}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(mean, mean * 0.12)):.2f}" for s in range(start, start + 200_000, 500)] # A male's single-copy chrY at half the depth; a female's chrX (two copies) at the full depth. sx = "Y" if SEX == "male" else "X" ymean = MEAN_DEPTH / (2 if SEX == "male" else 1) * rng.uniform(0.98, 1.0) for start, end in SINGLE_COPY_FIRST[sx]: s0 = (start // 500 + 1) * 500 lines += [f"chr{sx}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(ymean, ymean * 0.14)):.2f}" for s in range(s0, min(end, s0 + 1_000_000) - 500, 500)] put(path, "\n".join(lines) + "\n", gz=True) r = subprocess.run([og_binary(), "qc", "sex-loss", "--table", os.path.join(d, "qc", f"{SAMPLE}_sample_qc.tsv"), "--regions", path], stdout=subprocess.DEVNULL) if r.returncode: raise SystemExit(f"og qc sex-loss failed (exit {r.returncode})") def ancestry_fine(d, rng): """Step 26b's table (ADMIXTURE fractions with their bootstrap range, then the painted shares) and its painting, written by og ancestry-fine report from made-up ADMIXTURE and RFMix output. Mostly West Eurasian, as step 25's EUR placement above.""" continent = {"WEA": 0.942, "CSA": 0.031, "AFR": 0.012, "EAS": 0.008, "AMR": 0.005, "OCE": 0.002} region = {"NWE": 0.463, "IBE": 0.274, "ITA": 0.118, "NEE": 0.052, "SAR": 0.009, "CAU": 0.011, "LEV": 0.008, "BDN": 0.002, "NAF": 0.005, "CSA": 0.031, "WAF": 0.007, "EAF": 0.003, "CAF": 0.001, "SAF": 0.001, "EAS": 0.006, "SIB": 0.002, "AMR": 0.005, "OCE": 0.002} # GRCh38 lengths of chr1 to chr22. chrom_len = [248956422, 242193529, 198295559, 190214555, 181538259, 170805979, 159345973, 145138636, 138394717, 133797422, 135086622, 133275309, 114364328, 107043718, 101991189, 90338345, 83257441, 80373285, 58617616, 64444167, 46709983, 50818468] with tempfile.TemporaryDirectory() as work: model, adm, paint, out = (os.path.join(work, x) for x in ("model", "admixture", "paint", "out")) for x in (model, adm, paint): os.makedirs(x) levels = [] for level, fr in (("continent", continent), ("region", region)): codes = list(fr) levels.append(f"{level}\t{len(codes)}\t{','.join(codes)}\n") put(os.path.join(adm, f"{SAMPLE}_{level}.{len(codes)}.Q"), " ".join(f"{fr[c]:.6f}" for c in codes) + "\n") # 20 bootstrap replicates, each fraction moved by a few percent of its spread. boot = [] for _ in range(20): row = [max(0.0, fr[c] + rng.gauss(0, max(0.002, math.sqrt(fr[c] * (1 - fr[c])) * 0.06))) for c in codes] boot.append(" ".join(f"{v / sum(row):.6f}" for v in row) + "\n") put(os.path.join(adm, f"{SAMPLE}_{level}.boot.Q"), "".join(boot)) put(os.path.join(model, "levels.tsv"), "".join(levels)) # The painting: each chromosome in windows of 3 to 15 Mb, each haplotype # West Eurasian but for a few windows. groups = ["AFR", "CSA", "WEA"] for i, n in enumerate(chrom_len): c = f"chr{i + 1}" rows = ["#Subpopulation order/codes: " + "\t".join(f"{g}={k}" for k, g in enumerate(groups)), "#chm\tspos\tepos\tsgpos\tegpos\tn snps\t" + f"{SAMPLE}.0\t{SAMPLE}.1"] pos, end_of = rng.randint(200_000, 1_500_000), n - rng.randint(200_000, 1_500_000) while pos < end_of - 2_000_000: end = min(end_of, pos + rng.randint(3_000_000, 15_000_000)) haps = [2 if rng.random() < 0.85 else rng.choice([1, 1, 0]) for _ in (0, 1)] rows.append(f"{c}\t{pos}\t{end}\t{pos / 1e6:.3f}\t{end / 1e6:.3f}\t100\t{haps[0]}\t{haps[1]}") pos = end put(os.path.join(paint, f"{c}.msp.tsv"), "\n".join(rows) + "\n") r = subprocess.run([og_binary(), "ancestry-fine", "report", "--sample", SAMPLE, "--model", model, "--admixture", adm, "--paint", paint, "--panel", "demo", "--out", out], stdout=subprocess.DEVNULL) if r.returncode: raise SystemExit(f"og ancestry-fine report failed (exit {r.returncode})") dest = os.path.join(d, "ancestry_fine") os.makedirs(dest, exist_ok=True) for name in (f"{SAMPLE}_ancestry_fine.tsv", f"{SAMPLE}_painting.png"): shutil.copy(os.path.join(out, name), os.path.join(dest, name)) def paralogs(d): """Step 35b: a Paraphase JSON with the usual copy numbers (two of each gene, four in each gene family), as og paraphase table tables it. Paraphase reads PacBio HiFi; DEMO-001 carries the section so its card is in the pictures.""" def region(total, rsi, **kw): return dict({"total_cn": total, "genome_depth": MEAN_DEPTH, "region_depth": {"median": (total or 4) * 15.6}, "region_specific_info": rsi}, **kw) data = { "smn1": region(4, {"smn1_cn": 2, "smn2_cn": 2, "smn_del78_cn": 0}), "hba": region(4, {"genotype": "aa/aa", "sv_called": {}}, final_haplotypes={"1": "hba_hba2hap1", "2": "hba_hba2hap2", "3": "hba_hba1hap1", "4": "hba_hba1hap2"}), "rccx": region(4, {"annotated_alleles": ["WT", "WT"], "phasing_success": True}), "GBA": region(4, {}, genes_in_region=["GBA1", "GBAP1"], final_haplotypes={"1": "GBA_gene1hap1", "2": "GBA_gene1hap2", "3": "GBA_gene2hap1", "4": "GBA_gene2hap2"}), "pms2": region(4, {"gene_cn": 2}), "strc": region(4, {"gene_cn": 2}), "ncf1": region(6, {"gene_cn": 2}), "OTOA": region(4, {}, genes_in_region=["OTOA", "OTOAP1"], final_haplotypes={"1": "OTOA_gene1hap1", "2": "OTOA_gene1hap2", "3": "OTOA_gene2hap1", "4": "OTOA_gene2hap2"}), "CFC1": region(4, {}, genes_in_region=["CFC1", "CFC1B"]), "NEB": region(6, {}, genes_in_region=["NEB"]), "TNXB": region(None, {}, genes_in_region=["TNXB"]), } out = os.path.join(d, "paralogs") os.makedirs(out, exist_ok=True) with tempfile.TemporaryDirectory() as tmp: j = os.path.join(tmp, f"{SAMPLE}.paraphase.json") with open(j, "w") as f: json.dump(data, f) r = subprocess.run([og_binary(), "paraphase", "table", "--sample", SAMPLE, "--json", j, "--genes", os.path.join(REPO, "assets", "paraphase_genes.tsv"), "--out-cn", os.path.join(out, f"{SAMPLE}_paraphase_copy_number.tsv"), "--out-variants", os.path.join(out, f"{SAMPLE}_paraphase_variants.tsv")], stdout=subprocess.DEVNULL) if r.returncode: raise SystemExit(f"og paraphase table failed (exit {r.returncode})") # Step 35 (Parascopy, short reads): two SMN1 and two SMN2 copies at exon 7, # in the columns of its _smn_copy_number.tsv. put(os.path.join(out, f"{SAMPLE}_smn_copy_number.tsv"), "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\nchr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t2,2\t33,30\t" "chr5:70049000-70077000:-\n") # Step 35's HBA1/HBA2 table: four alpha-globin genes. put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), HBA_TABLE) def couple(d, rng): """Step 39 with an invented partner, DEMO-002: each carries one change in a different recessive gene of the carrier list, so no gene is shared and the card names none. Rows by og couple screen, fed a made-up ClinVar screen of each partner. Only DEMO-001 carries the table: the family page reads one copy of it.""" if SAMPLE != "DEMO-001": return genes_tsv = os.path.join(REPO, "assets", "carrier_genes.tsv") with open(genes_tsv) as f: lines = [x.rstrip("\n").split("\t") for x in f if x.strip() and not x.startswith("#")] genes = {c[0]: dict(zip(lines[0], c)) for c in lines[1:]} ar = sorted(g for g, v in genes.items() if v["inheritance"] == "AR" and "ACMG" in v["lists"]) with tempfile.TemporaryDirectory() as work: args = [] for who, gene in zip((SAMPLE, "DEMO-002"), rng.sample(ar, 2)): g = genes[gene] pos = rng.randint(int(g["start"]), int(g["end"])) ref, alt = rng.sample("ACGT", 2) put(os.path.join(work, f"{who}.vcf"), "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" f"{g['chrom']}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\tGENEINFO={gene}:1;CLNSIG=Pathogenic;" "CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\tGT\t0/1\n") args += ["--clinvar", f"{who}={os.path.join(work, who + '.vcf')}"] put(os.path.join(work, "pedigree.tsv"), "sample\tsex\tfather\tmother\tpartner\n" f"{SAMPLE}\t{SEX}\t.\t.\tDEMO-002\nDEMO-002\tfemale\t.\t.\t{SAMPLE}\n") r = subprocess.run([og_binary(), "couple", "screen", "--pedigree", os.path.join(work, "pedigree.tsv"), "--genes", genes_tsv, "--par-bed", os.path.join(REPO, "assets", "par_grch38.bed"), *args, "--out-dir", work], stdout=subprocess.DEVNULL) if r.returncode != 0: raise SystemExit(f"og couple screen failed (exit {r.returncode})") os.makedirs(os.path.join(d, "couple"), exist_ok=True) shutil.copyfile(os.path.join(work, f"{SAMPLE}_couple_carrier.tsv"), os.path.join(d, "couple", f"{SAMPLE}_couple_carrier.tsv")) def consumer_export(d): """Step 40's summary table, in og consumer export's rows: the pinned site table keeps 614,138 of the chip list's 638,463 sites (docs/40-export-consumer.md); the counts below are made up.""" sites, from_blocks, from_records = 614138, 470912, 140206 reasons = {"no gVCF record": 1954, "inside a deletion or a longer allele": 611, "filtered call": 309, "no-call in the gVCF": 146} called = from_blocks + from_records assert sites - called == sum(reasons.values()) rows = [("sites", sites), ("called", called), ("no_call", sites - called), ("from_reference_blocks", from_blocks), ("from_variant_records", from_records), ("reference_checked", from_records), ("reference_mismatch", 0), ("template_sites", 638463), ("table_dropped", 24325), ("sex", SEX), ("file_23andme", f"{SAMPLE}_23andme_v5.zip"), ("file_ancestrydna", f"{SAMPLE}_ancestrydna_layout.zip")] rows += sorted((f"no_call: {k}", v) for k, v in reasons.items()) put(os.path.join(d, "consumer_export", f"{SAMPLE}_consumer_export.tsv"), "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows)) # Step 41's rows (og bloodtype report's columns and spelling), a common # European set. BLOODTYPE = [ ("ABO", "A1", "A1/O.01", "high", ""), ("RhD", "D+", "RHD/RHD (RHD copies by read depth 1.96)", "high", "weak and partial D (RHD missense alleles) are not assessed"), ("RhCE", "C+c+ E-e+", "C/c e/e", "high", ""), ("Kell", "K-k+ Kp(a-b+) Js(a-b+)", "k/k Kpb/Kpb Jsb/Jsb", "high", ""), ("Duffy", "Fy(a+b+)", "FY*A/FY*B", "high", ""), ("Kidd", "Jk(a+b+)", "Jka/Jkb", "high", ""), ("MNS", "M+N+ S-s+", "M/N s/s", "high", ""), ("Lutheran", "Lu(a-b+)", "Lub/Lub", "high", ""), ("Diego", "Di(a-b+)", "Dib/Dib", "high", ""), ("Dombrock", "Do(a+b+)", "Doa/Dob", "medium", ""), ("Colton", "Co(a+b-)", "Coa/Coa", "high", ""), ("Vel", "Vel+", "Vel/Vel", "high", ""), ("Langereis", "Lan+", "Lan/Lan", "high", ""), ("Junior", "Jr(a+)", "Jra/Jra", "high", ""), ("H", "H+", "H/H", "high", ""), ("Secretor", "secretor", "Se/se", "high", "null allele: FUT2 c.461G>A x1; weak secretor alleles (such as FUT2 c.418A>T, also written c.385A>T) " "and a deletion of the whole FUT2 gene are not assessed"), ("HPA-1", "HPA-1(a+b-)", "1a/1a", "high", ""), ("HPA-2", "HPA-2(a+b-)", "2a/2a", "high", ""), ("HPA-3", "HPA-3(a+b+)", "3a/3b", "high", ""), ("HPA-4", "HPA-4(a+b-)", "4a/4a", "high", ""), ("HPA-5", "HPA-5(a+b-)", "5a/5a", "high", ""), ("HPA-6", "HPA-6(a+b-)", "6a/6a", "high", ""), *[(f"HPA-{n}", f"HPA-{n}(a+b-)", f"{n}a/{n}a", "high", "") for n in range(7, 15)], ("HPA-15", "HPA-15(a+b+)", "15a/15b", "high", ""), *[(f"HPA-{n}", f"HPA-{n}(a+b-)", f"{n}a/{n}a", "high", "") for n in range(16, 36)], ("HNA-1", "HNA-1(a+b+c-)", "FCGR3B*01/FCGR3B*02", "low", "FCGR3B and FCGR3A are near copies and the FCGR3B copy number varies (none to four or more): " "short reads can misplace these calls, the copies are not counted, and HNA-1null (no FCGR3B) " "is not assessed"), ("HNA-3", "HNA-3(a+b-)", "3a/3a", "high", ""), ("HNA-4", "HNA-4(a+b-)", "4a/4a", "high", ""), ("HNA-5", "HNA-5(a+b-)", "5a/5a", "high", ""), ] def bloodtype(d): names = subprocess.run([og_binary(), "bloodtype", "systems"], capture_output=True, text=True, check=True).stdout.split("\n")[:-1] if [r[0] for r in BLOODTYPE] != names: raise SystemExit(f"BLOODTYPE lists {[r[0] for r in BLOODTYPE]}, step 41 writes {names}") # A platelet or neutrophil antigen the phenotype lacks gets the marker table's # meaning at the head of the note, as og bloodtype report writes it. with open(os.path.join(REPO, "assets", "bloodtype_markers.tsv")) as f: lines = [ln for ln in f if not ln.startswith("#") and ln.strip()] means = {} for r in csv.DictReader(lines, delimiter="\t"): if r.get("meaning"): means.setdefault(r["system"], r["meaning"]) rows = [] for system, phenotype, genotype, conf, note in BLOODTYPE: lacks = re.findall(r"([a-z])-", phenotype.partition("(")[2]) if system in means and lacks: prefix, num = system.split("-") head = f"lacks {', '.join(f'{prefix}-{num}{a}' for a in lacks)}: {means[system]}" note = "; ".join([head] + ([note] if note else [])) rows.append((system, phenotype, genotype, conf, note)) os.makedirs(os.path.join(d, "bloodtype"), exist_ok=True) with open(os.path.join(d, "bloodtype", f"{SAMPLE}_bloodtype.tsv"), "w", newline="") as f: w = csv.writer(f, delimiter="\t", lineterminator="\n") w.writerow(["system", "phenotype", "genotype", "confidence", "note"]) w.writerows(rows) # Step 42: copies of each row's alt allele, a common European set; the # TAS2R38 rows are one taster and one non-taster haplotype, APOE is e3/e3. # AHR and OR2M7 have no genotype, as a gap in the gVCF. TRAIT_ALT_COPIES = { "rs4988235": 1, "rs182549": 1, "rs145946881": 0, "rs41380347": 0, "rs41525747": 0, "rs671": 0, "rs1229984": 2, "rs762551": 2, "rs2472297": 1, "rs713598": 1, "rs1726866": 1, "rs10246939": 1, "rs17822931": 0, "rs10427255": 1, "rs1815739": 1, "rs12913832": 1, "rs1800407": 0, "rs16891982": 2, "rs1426654": 0, "rs12203592": 0, "rs1805007": 0, "rs1805008": 0, "rs1805009": 0, "rs3827760": 0, "rs1801133": 1, "rs601338": 1, "rs2282679": 1, "rs887829": 1, "rs1801272": 0, "rs429358": 0, "rs7412": 0, "rs10455872": 0, "rs3798220": 0, "rs78655421": 0, } def y_str(d, rng): """Step 45: one row per marker of assets/ystr_panel.tsv in the columns of og y str-report. The multi-copy, absent and too-long markers (a repeat with its 10-base flanks longer than 150 bp reads) read not callable, with og's notes, as in a real run; every other marker gets a made-up count at confidence high.""" with open(os.path.join(REPO, "assets", "ystr_panel.tsv"), encoding="utf-8") as f: lines = [x.rstrip("\n").split("\t") for x in f if x.strip() and not x.startswith("#")] cols = ["order", "panel", "marker", "count", "confidence", "reads", "stutter_reads", "flank_indel_reads", "q", "note"] out = ["\t".join(cols) + "\n"] for c in lines[1:]: r = dict(zip(lines[0], c)) row = dict.fromkeys(cols, "") row.update(order=r["order"], panel=r["panel"], marker=r["marker"], confidence="not callable") spans = [[int(x) for x in part.split("/")[0].split(":")[1].split("-")] for part in ([] if r["regions"] == "." else r["regions"].split(";"))] if r["kind"] == "span": # one region over the whole locus spans = [[spans[0][0], spans[-1][1]]] longest = max((e - b + 1 for b, e in spans), default=0) if r["kind"] == "multicopy": row["note"] = "not callable from short reads: a multi-copy marker in duplicated Y sequence" elif r["kind"] == "absent": row["note"] = "not callable: no region for it in the HipSTR marker set" elif longest + 2 * 10 > 150: row["note"] = (f"not callable from short reads: a {longest} bp repeat is longer than 150 bp reads can " f"span with 10 bases on each side") else: row.update(count=str(rng.randint(10, 30)), confidence="high", reads=str(rng.randint(18, 34)), stutter_reads="0", flank_indel_reads="0", q="0.99", note=".") out.append("\t".join(row[k] for k in cols) + "\n") put(os.path.join(d, "y_str", f"{SAMPLE}_y_str.tsv"), "".join(out)) def viral(d): """Step 47: what og viral report writes for made-up coverage: no HHV-6, 7 Epstein-Barr reads (under the 10 reads of a call), and the lab's phiX control, at a human depth of 31.""" head = ("virus\taccession\tlength\treads\tbreadth\tmean_depth\tmode_depth\thuman_depth\trelative_depth" "\tcopies_per_cell\tcall\tnote\n") rows = [("HHV-6A", "NC_001664.4", "159378", "0", "0.0000", "0.000", "0", "0.000", ".", "not found", ""), ("HHV-6B", "NC_000898.1", "162114", "0", "0.0000", "0.000", "0", "0.000", ".", "not found", ""), ("Human gammaherpesvirus 4", "NC_007605.1", "171823", "7", "0.0041", "0.006", "0", "0.000", ".", "below threshold", "7 counted reads, under the 10 a call needs: too few to tell the virus from " "contamination at this depth."), ("Escherichia phage phiX174", "NC_001422.1", "5386", "1412", "1.0000", "38.200", "37", "1.194", "2.4645", "lab control (phiX)", "Illumina adds phiX to the run as a control: these reads were added in the lab.")] put(os.path.join(d, "viral", f"{SAMPLE}_viral.tsv"), head + "".join("\t".join(r[:7] + ("31.00",) + r[7:]) + "\n" for r in rows)) # Step 35's HBA1/HBA2 depths (mean depth per region of the HBA row of # bin/cn_loci.tsv, every read) and the share of each left at MAPQ 1 or more, # and the table og cn call --locus HBA writes for them: four genes. HBA_DEPTH = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} HBA_Q1 = {"flank_left": 0.98, "HBA2": 0.76, "HBA1": 0.76, "flank_right": 0.98} HBA_TABLE = ("metric\tvalue\nstatus\tcalled\nalpha_globin_genes\t4\nestimate\t3.96\nhba2_depth\t30.9\n" "hba1_depth\t30.1\nflank_left_depth\t31.2\nflank_right_depth\t30.4\nmapq0_share\t0.240\n" "message\tfour alpha-globin genes (aa/aa): no deletion seen\n") def lpa_depth(row): """Step 46's made-up depths over the LPA_KIV2 row's segments (catalog order, from bin/copy_number.py's load_catalog): every read, and MAPQ 1 or more. About 37 KIV-2 repeats at a flank depth near 31, half the KIV-2 depth at MAPQ 0.""" m = {n: 86.5 * (1.02 if i % 2 == 0 else 0.98) for i, (n, *_) in enumerate(row["_targets"])} m.update({n: 31.0 * (1.03 if i % 2 == 0 else 0.97) for i, (n, *_) in enumerate(row["_controls"])}) return m, {n: v * (0.5 if n.startswith("kiv2_") else 0.98) for n, v in m.items()} # The table og cn call --locus LPA_KIV2 writes for lpa_depth. LPA_TABLE = ("metric\tvalue\nstatus\testimated\nkiv2_copies\t37.0\nci95_low\t36.3\nci95_high\t37.8\n" "class\ttypical\nkiv2_depth\t86.61\nflank_depth\t31.09\nmapq0_share\t0.500\n" "message\tabout 37 KIV-2 repeats in total (95% interval 36.3 to 37.8): within or overlapping the range " "most people have (30 to 45 for about 70% of the 1000 Genomes samples)\n" "advice\tA research estimate from read depth, not a diagnostic test. Lp(a) is measured in blood; a 2022 " "European Atherosclerosis Society consensus advises measuring it at least once in every adult.\n") def lpa(d, rng): """Step 46: the table og cn call writes for lpa_depth (LPA_TABLE).""" put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), LPA_TABLE) CHIP_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", "alt_reads", "depth", "filter", "status", "class", "note"] CHIP_DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] def chip_limit(depth, k=3, power=0.8): """og chip's detection limit: the smallest VAF that gives K or more reads with the change with probability POWER at DEPTH (binomial, by bisection).""" n = int(depth) def p_at_least(f): return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) lo, hi = 0.0, 1.0 for _ in range(60): mid = (lo + hi) / 2 lo, hi = (lo, mid) if p_at_least(mid) >= power else (mid, hi) return hi def chip(d, rng): """Step 49: no CHIP call, and one depth row per gene of assets/chip_genes.tsv at a made-up mean depth near 31, with og chip's detection limit.""" with open(os.path.join(REPO, "assets", "chip_genes.tsv"), encoding="utf-8") as f: lines = [x for x in f if not x.startswith("#") and x.strip()] rows = [] for gene in (x.split("\t")[0] for x in lines[1:]): mean = rng.uniform(29.0, 33.0) rows.append({"gene": gene, "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_limit(mean):.3f}", "status": "covered"}) calls = [] if has("chip"): # One small DNMT3A clone, above the detection limit at this depth (og chip's columns). calls.append({"gene": "DNMT3A", "chrom": "chr2", "pos": "25234373", "ref": "C", "alt": "T", "consequence": "missense_variant", "protein_change": "p.R882H", "exon": "23", "vaf": "0.161", "alt_reads": "8", "depth": "50", "filter": "PASS", "status": "chip", "class": "hotspot", "note": "a published CHIP hotspot"}) put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(CHIP_COLUMNS) + "\n" + "".join("\t".join(c[k] for k in CHIP_COLUMNS) + "\n" for c in calls)) put(os.path.join(d, "chip", f"{SAMPLE}_chip_depth.tsv"), "\t".join(CHIP_DEPTH_COLUMNS) + "\n" + "".join("\t".join(x[c] for c in CHIP_DEPTH_COLUMNS) + "\n" for x in rows)) def archaic(d, rng): """Step 43: bin/archaic_report.py's key/value summary, with made-up shares near the published non-African range and five made-up longest segments.""" analysed = 2_870_000_000 nea_n, den_n = rng.randint(420, 520), rng.randint(2, 6) nea_bp, den_bp = int(analysed * rng.uniform(0.017, 0.021)), int(analysed * rng.uniform(0.0002, 0.0006)) rows = [("sample", SAMPLE), ("method", "IBDmix"), ("archaic_genomes", "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan"), ("population", "EUR"), ("min_lod", "4"), ("min_length_bp", "50000"), ("analysed_bp", analysed), ("analysed_chromosomes", 22), ("analysed_sites", rng.randint(5_500_000, 6_500_000)), ("genes", "GENCODE protein-coding"), ("neanderthal_segments", nea_n), ("neanderthal_bp", nea_bp), ("neanderthal_percent", f"{100 * nea_bp / analysed:.2f}"), ("denisovan_segments", den_n), ("denisovan_bp", den_bp), ("denisovan_percent", f"{100 * den_bp / analysed:.2f}")] segs = [("chr1", "Neanderthal"), ("chr3", "Neanderthal"), ("chr6", "Neanderthal"), ("chr9", "Neanderthal"), ("chr12", "Denisovan")] longest = [] for i, (c, anc) in enumerate(segs, 1): s = rng.randint(10_000_000, 90_000_000) n = int(rng.uniform(0.9, 1.3 - 0.08 * i) * 1e6) who = "Altai Denisovan" if anc == "Denisovan" else "Altai/Vindija" longest.append((f"longest_{i}", f"{c}:{s}-{s + n} {anc} {n / 1e6:.2f} Mb ({who}); genes: none")) put(os.path.join(d, "archaic", f"{SAMPLE}_archaic.tsv"), "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows + longest)) def ancient(d, rng): """Step 44: what og ancient report writes (the nearest-group table, the info table, the text and the PC plot) for a made-up panel: present-day people around the origin, ancient people of a few AADR-style group labels each around its own point, and the person among them.""" groups = [("England_BellBeaker", 4200, "United Kingdom", "Amesbury Down"), ("Germany_CordedWare", 4600, "Germany", "Esperstedt"), ("Spain_BellBeaker", 4200, "Spain", "Camino de las Yeseras"), ("Czechia_EBA_Unetice", 3900, "Czech Republic", "Moravia"), ("Hungary_LBA", 3250, "Hungary", "Tiszafüred"), ("Italy_Imperial_Rome", 1800, "Italy", "Rome"), ("Sweden_Viking", 1030, "Sweden", "Sigtuna"), ("France_Neolithic", 5700, "France", "Gurgy"), ("Turkey_Neolithic", 8200, "Turkey", "Barcın"), ("Russia_Yamnaya_Samara", 4950, "Russia", "Samara")] person = (0.01234, -0.00456) evals = "#eigvals: 9.000 4.000 2.000 1.000\n" moderns = [f"m{i} " + " ".join(f"{rng.gauss(0.0, sd):.6f}" for sd in (0.03, 0.025, 0.02, 0.015)) + f" {('French', 'Sardinian', 'Basque')[i % 3]}\n" for i in range(300)] ancients, anc_rows = [], ["id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"] for i, (g, bp, country, loc) in enumerate(groups): spread = 0.004 + 0.006 * i cx, cy = person[0] + rng.uniform(-spread, spread), person[1] + rng.uniform(-spread, spread) for j in range(rng.randint(3, 12)): aid = f"a{i}_{j}" ancients.append(f"{aid} {cx + rng.gauss(0, 0.004):.6f} {cy + rng.gauss(0, 0.004):.6f} 0.000000 0.000000 Ancient\n") date = bp + rng.randint(-150, 150) anc_rows.append(f"{aid}\t{g}\t{date}\t50\t.\t{loc}\t{country}\t50\t10\t500000\n") with tempfile.TemporaryDirectory() as work: paths = {n: os.path.join(work, n) for n in ("panel.evec", "run.evec", "ancients.tsv", "stats.tsv")} put(paths["panel.evec"], evals + "".join(moderns + ancients)) put(paths["run.evec"], evals + "".join(moderns) + f"{SAMPLE} {person[0]:.6f} {person[1]:.6f} 0.000000 0.000000 Person\n") put(paths["ancients.tsv"], "".join(anc_rows)) put(paths["stats.tsv"], f"panel_sites\t597573\nsites_genotyped\t560000\nother_allele\t120\n" f"sites_used\t{rng.randint(520_000, 560_000)}\n") out = os.path.join(d, "ancient") os.makedirs(out, exist_ok=True) r = subprocess.run([og_binary(), "ancient", "report", "--panel-evec", paths["panel.evec"], "--run-evec", paths["run.evec"], "--ancients", paths["ancients.tsv"], "--sample", SAMPLE, "--release", "v62.0", "--stats", paths["stats.tsv"], "--out-tsv", os.path.join(out, f"{SAMPLE}_ancient.tsv"), "--out-info", os.path.join(out, f"{SAMPLE}_ancient_info.tsv"), "--out-txt", os.path.join(out, f"{SAMPLE}_ancient.txt"), "--out-svg", os.path.join(out, f"{SAMPLE}_ancient_pca.svg")], stdout=subprocess.DEVNULL) if r.returncode: sys.exit("ERROR: og ancient report failed on the made-up panel") def exomiser(d): """Step 48: what og exomiser report writes for made-up Exomiser rows for two searched symptoms: two weak candidates of uncertain significance, the shape of a search that finds nothing to act on. DEMO-003's search also ranks the planted gene (PLANTED) third, at the planted hit: a symptom search can rank the gene of a result his page keeps closed.""" terms = [("HP:0001250", "Seizure"), ("HP:0002315", "Headache")] # rank, gene, (exomiser, phenotype, variant, p), variant, HGVS, disease ranks = [(1, "SCN1A", ("0.412", "0.788", "0.534", "0.021"), "2:165991230 C>T", "SCN1A:ENST00000674923.1:c.3199G>A:p.(Ala1067Thr)", "Dravet syndrome (OMIM:607208)"), (2, "CACNA1A", ("0.297", "0.701", "0.421", "0.048"), "19:13299371 G>A", "CACNA1A:ENST00000360228.11:c.2134C>T:p.(Arg712Cys)", "Migraine, familial hemiplegic, 1 (OMIM:141500)")] if has("act"): c, pos, ref, alt = PLANTED["hit"] ranks.append((3, PLANTED["gene"], ("0.205", "0.512", "0.399", "0.061"), f"{c.removeprefix('chr')}:{pos} {ref}>{alt}", f"{PLANTED['gene']}:ENST00000262186.10:c.1C>T:p.(Met1?)", "Long QT syndrome 2 (OMIM:613688)")) freq = "GNOMAD_E_NFE 0.0123%" top = ["rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant_score\tp_value\tvariant\tgenotype" "\tconsequence\thgvs\tacmg\tacmg_evidence\tdisease\tclinvar\tmax_frequency\n"] words = ["Symptoms searched: " + ", ".join(f"{label} ({t})" for t, label in terms) + "."] for rank, gene, sc, var, hgvs, disease in ranks: top.append("\t".join([str(rank), gene, "AD", *sc, var, "0/1", "missense_variant", hgvs, "UNCERTAIN_SIGNIFICANCE", ".", disease, ".", freq]) + "\n") words.append(f"{rank}. {gene} (autosomal dominant): Exomiser score {sc[0]} (symptom match {sc[1]}, variant " f"{sc[2]}). {var} heterozygous, missense_variant {hgvs}, ACMG uncertain significance, linked " f"disease {disease}, highest population frequency {freq}.") words.append("A ranking to bring to a clinical geneticist, not a diagnosis. A high score means a rare variant, " "predicted to be damaging, in a gene linked to symptoms like the ones given; it does not show that " "the variant causes them. A low score or an empty list does not rule a genetic cause out: this " "search reads coding and splice variants against the terms given, and not structural variants, " "repeat expansions, mitochondrial heteroplasmy or variants outside genes.") out = os.path.join(d, "exomiser") put(os.path.join(out, f"{SAMPLE}_exomiser_terms.tsv"), "hpo_id\tlabel\tgiven_as\n" + "".join(f"{t}\t{label}\t{t}\n" for t, label in terms)) put(os.path.join(out, f"{SAMPLE}_exomiser_top.tsv"), "".join(top)) put(os.path.join(out, f"{SAMPLE}_exomiser.txt"), "\n".join(words) + "\n") def traits(d): """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by og traits table.""" with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: rows = list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) unknown = sorted(r["rsid"] for r in rows if r["rsid"] not in TRAIT_ALT_COPIES | {"rs4410790": 0, "rs4481887": 0}) if unknown: raise SystemExit(f"assets/traits.tsv has rows TRAIT_ALT_COPIES does not give: {', '.join(unknown)}") os.makedirs(os.path.join(d, "traits"), exist_ok=True) with tempfile.TemporaryDirectory() as work: genotypes = os.path.join(work, "g.tsv") with open(genotypes, "w") as f: for r in rows: if r["rsid"] in TRAIT_ALT_COPIES: n = TRAIT_ALT_COPIES[r["rsid"]] gt = "/".join([r["ref"]] * (2 - n) + [r["alt"]] * n) f.write(f"{r['chrom']}\t{r['pos']}\t{r['ref']}\t{r['alt']}\t{gt}\t60\n") subprocess.run([os.environ.get("OG", os.path.join(REPO, "bin", "og")), "traits", "table", "--table", os.path.join(REPO, "assets", "traits.tsv"), "--rules", os.path.join(REPO, "assets", "genotype_rules.tsv"), "--genotypes", genotypes, "--sample", SAMPLE, "--out", os.path.join(d, "traits", f"{SAMPLE}_traits.tsv"), "--rules-out", os.path.join(work, "rules.tsv")], check=True) def mocha(d): """Step 51: what og mocha report writes for made-up MoChA tables of a 30x genome (chr1 to chr22 read, 95,000 hets per chromosome of 100 Mb, the sample filter met), with no mosaic call; DEMO-002's carries a loss of 20q in about 12% of the blood cells, which MoChA's filters keep and the floor lets through.""" calls = ("chrom\tstart\tend\tlength_mb\tarms\ttype\tcell_fraction\tbdev\trel_cov\tn_hets\tlod_baf_phase" "\tlod_lrr_baf\treported\treason\n") if has("mocha"): calls += "chr20\t30100000\t42300000\t12.20\t20q\tloss\t0.120\t0.0310\t1.8600\t4210\t48.2\t35.1\tyes\t.\n" n = "1" if has("mocha") else "0" summary = [("status", "assessed"), ("message", "."), ("depth", "31.0"), ("het_sites", "2090000"), ("phased_het_sites", "2068000"), ("het_per_10mb", "9500"), ("analysed_mb", "2200.0"), ("chromosomes", ",".join(f"chr{i}" for i in range(1, 23))), ("call_rate", "0.9981"), ("baf_auto", "0.0124"), ("floor_read", "0.011"), ("floor_min", "0.05"), ("floor", "0.050"), ("calls_mocha", n), ("calls_reported", n), ("phasing", "beagle")] put(os.path.join(d, "mosaic", f"{SAMPLE}_mocha.tsv"), calls) put(os.path.join(d, "mosaic", f"{SAMPLE}_mocha_summary.tsv"), "metric\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in summary)) def vntyper(d): """Step 52: what og vntyper report writes for a made-up VNtyper run with no MUC1 frameshift, at a VNTR depth of 212x that its coverage check passes.""" rows = [("status", "negative"), ("confidence", "Negative"), ("variant", "."), ("flag", "."), ("alt_depth", "."), ("region_depth", "."), ("depth_score", "."), ("vntr_mean_depth", "212.40"), ("vntr_uncovered_pct", "0.00"), ("coverage_qc", "PASS"), ("sensitivity", "Mean depth over the MUC1 VNTR: 212x. At that depth (at 100x or more) VNtyper found the " "known variant in 160 of 161 runs (99%), in VNtyper 2.0.47's downsampling of 42 confirmed positive " "GRCh38 exomes (repeated runs, not independent tests)."), ("message", "VNtyper calls no MUC1 VNTR frameshift."), ("advice", "A negative here lowers the chance of the frameshifts VNtyper looks for; it does not rule " "ADTKD-MUC1 out. A family history of kidney failure with gout or a bland urine sediment still deserves " "a clinical test (SNaPshot for dupC).")] put(os.path.join(d, "vntyper", f"{SAMPLE}_vntyper.tsv"), "metric\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows)) def locityper(d, rng): """Step 53: what og locityper report writes for a made-up Locityper run that called every locus of assets/locityper_loci.bed with two HPRC panel haplotypes, made-up qualities and read counts, and passed the authors' filter (no warning, a weighted distance under 30, few unexplained reads).""" panel = ["HG00438.1", "HG00733.2", "HG01891.1", "HG02257.2", "NA18906.1", "NA19240.2"] out = ["locus\tregion\tstatus\tgenotype\thaplotype_1\thaplotype_2\tquality\ttotal_reads\tunexplained_reads" "\tweighted_distance\twarnings\tfilter\treason\n"] with open(os.path.join(REPO, "assets", "locityper_loci.bed")) as f: for line in f: if not line.strip() or line.startswith("#"): continue chrom, start, end, name = line.rstrip("\n").split("\t")[:4] haps = rng.sample(panel, 2) quality, total = round(rng.uniform(25, 60), 1), rng.randint(800, 4000) unexpl, dist = rng.randint(0, 30), round(rng.uniform(0.1, 3), 3) out.append("\t".join([name, f"{chrom}:{int(start) + 1}-{end}", "called", ",".join(haps), *haps, f"{quality:.1f}", str(total), str(unexpl), f"{dist:.3f}", ".", "pass", "."]) + "\n") put(os.path.join(d, "locityper", f"{SAMPLE}_locityper.tsv"), "".join(out)) def mei(d, rng): """Step 50: what og mei report writes for a made-up MEGAnE VCF (two Alu and one L1 insertion, away from the catalog) and a made-up depth near 31x over every window of assets/mei_sites.tsv, so every catalog row reads "not seen".""" calls = ["chrom\tpos\tend\telement\tsubfamily\tcopies\tfilter\tconfidence\tgenes\tlocation\tbenign_af" "\tacmg_class\n"] for c, p, e, sub in (("chr3", 41258120, "Alu", "AluYa5"), ("chr7", 102931455, "Alu", "AluYb8"), ("chr11", 63044871, "L1", "L1HS")): calls.append(f"{c}\t{p}\t{p}\t{e}\t{sub}\t1\tPASS\tH\t.\t.\t.\t.\n") with open(os.path.join(REPO, "assets", "mei_sites.tsv")) as f: lines = [x.rstrip("\n") for x in f if x.strip() and not x.startswith("#")] sites = [lines[0] + "\tstatus\tmean_depth\tcall\n"] for line in lines[1:]: start, end = (int(x) for x in line.split("\t")[2:4]) mean = round((end - start) * rng.uniform(28, 34)) / (end - start) sites.append(f"{line}\tnot seen\t{mean:.1f}\t.\n") put(os.path.join(d, "mei", f"{SAMPLE}_mei.tsv"), "".join(calls)) put(os.path.join(d, "mei", f"{SAMPLE}_mei_sites.tsv"), "".join(sites)) def run_status(d, started): """logs/run_status.tsv as run-all.sh writes it, every step ok.""" lines = ["# run-all.sh: when this run started and how each step ended", f"meta\tstarted_epoch\t{int(started)}", f"meta\tstarted_utc\t{time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(started))}", f"meta\tdeclared_sex\t{SEX}"] lines += [f"step\t{s}\tok" for s in sorted(collect_summary.RUN_ALL_STEPS)] put(os.path.join(d, "logs", "run_status.tsv"), "\n".join(lines) + "\n") # --- the two commands ---------------------------------------------------------------- def make(a): global SAMPLE, SEX SAMPLE, SEX = a.sample, a.sex if not re.match(r"^DEMO-[0-9]{3}$", SAMPLE): raise SystemExit(f"--sample {SAMPLE}: an invented name of the form DEMO-001") # A fixed start, so the report's "Latest run" line is the same on every # retake; the files written now are newer, so none is marked stale. started = RUN_STARTED d = os.path.join(a.genome_dir, SAMPLE) if os.path.exists(d): raise SystemExit(f"{d} exists: give an empty --genome-dir") os.makedirs(d) run_status(d, started) # Each part gets its own generator, so changing one leaves the others' values alone. parts = [("variants", lambda r: variants(d, r, a.records)), ("clinvar", lambda r: clinvar(d, a.genome_dir)), ("pgx", lambda r: pgx(d, a.genome_dir, r)), ("prs", lambda r: prs(d, a.genome_dir, r)), ("coverage", lambda r: coverage(d, r)), ("sample_qc", lambda r: sample_qc(d)), ("reads", lambda r: reads(d, r)), ("structural", lambda r: structural(d, r)), ("repeats", lambda r: repeats_and_more(d, r)), ("clinical", lambda r: clinical(d, r)), # after "reads" and "sample_qc", whose files these two read ("read_count", lambda r: read_count(d)), ("mosaic", lambda r: mosaic(d, r)), ("ancestry_fine", lambda r: ancestry_fine(d, r)), ("paralogs", lambda r: paralogs(d)), ("couple", lambda r: couple(d, r)), ("consumer_export", lambda r: consumer_export(d)), ("bloodtype", lambda r: bloodtype(d)), ("traits", lambda r: traits(d)), ("y_str", lambda r: y_str(d, r) if SEX == "male" else None), ("viral", lambda r: viral(d)), ("lpa", lambda r: lpa(d, r)), ("chip", lambda r: chip(d, r)), ("archaic", lambda r: archaic(d, r)), ("ancient", lambda r: ancient(d, r)), ("exomiser", lambda r: exomiser(d)), ("mocha", lambda r: mocha(d)), ("vntyper", lambda r: vntyper(d)), ("mei", lambda r: mei(d, r)), ("locityper", lambda r: locityper(d, r)), ("callability", lambda r: callability(d, r))] for name, fn in parts: fn(random.Random(f"{a.seed}:{name}")) print(f" {name}: written") print(f"{SAMPLE} written in {d}") return 0 def check(a): """Fail when a section of step 24's summary is not ok.""" path = os.path.join(a.genome_dir, SAMPLE, "summary.json") with open(path) as f: s = json.load(f) bad = [(k, v["state"], v.get("note") or "") for k, v in s["sections"].items() if v["state"] != "ok"] for k, v in s["sections"].items(): print(f" {k:<14} {v['state']}") if s["cyp2d6"].get("agree") is not True: bad.append(("cyp2d6", "callers do not agree", json.dumps(s["cyp2d6"]["calls"]))) if bad: for k, st, note in bad: print(f"FAIL: {k} is {st} {note}".rstrip(), file=sys.stderr) print("A card would say 'Not run' or worse: feed that section in tests/demo/make_demo_sample.py.", file=sys.stderr) return 1 print(f"Every section of {path} is ok.") return 0 def main(argv=None): argv = sys.argv[1:] if argv is None else argv if argv[:1] == ["check"]: ap = argparse.ArgumentParser(prog="make_demo_sample.py check") ap.add_argument("--genome-dir", required=True) return check(ap.parse_args(argv[1:])) ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--genome-dir", required=True, help="an empty folder; DEMO-001 is written inside it") ap.add_argument("--seed", type=int, default=1) ap.add_argument("--records", type=int, default=4_700_000, help="variant records in the VCF (default 4700000)") ap.add_argument("--sample", default="DEMO-001", help="the invented sample to write (default DEMO-001)") ap.add_argument("--sex", default="male", choices=("male", "female")) return make(ap.parse_args(argv)) if __name__ == "__main__": sys.exit(main())