#!/usr/bin/env python3 """make_demo_sample.py: write DEMO-001, the invented sample of the docs pictures. make_demo_sample.py --genome-dir DIR [--seed 1] [--records 4700000] [--sample DEMO-001] [--sex male] make_demo_sample.py check --genome-dir DIR The first form writes DIR/DEMO-001 with every file steps 24, 27 and 28 read, as the steps that make them would have written them, so that every card of the reports shows a value. Nothing is read from a real sample: the values come from a random generator with a fixed seed and from a few hand-picked numbers below. It also writes the three reference files the run manifest reads (DIR/clinvar, DIR/t1k_idx, DIR/prs_scores), cut down to their header lines. Borrowed, not rewritten: - the PharmCAT report starts from PharmCAT's own example report (tests/fixtures/pharmcat/pharmcat-docs-example.json, PharmCAT's invented example sample); HLA-A, HLA-B and CYP2D6 become the outside calls step 36 passes on for DEMO-001, worked out by bin/pgx_outside_calls.py itself; - T1K's genotype table and the CYP2D6 depth check are the invented fixtures of tests/fixtures/pgx; - step 33's table is written by `collect_summary.py sample-qc`, and step 25's PRS and ancestry tables by `collect_summary.py prs-format` and `prs-table`, from made-up somalier, VerifyBamID2 and pgsc_calc files; - step 33's mosaic rows come from bin/mosaic_sex_loss.py, step 26b's painting from bin/ancestry_fine_report.py, step 35b's table from bin/paraphase_report.py, step 39's from bin/couple_report.py and step 42's from bin/traits_report.py, each fed made-up inputs; step 45's rows come from bin/ystr_report.py's marker_row, step 46's from bin/lpa_kiv2.py's assess, step 35's HBA1/HBA2 table from bin/hba_copy_number.py's assess, step 49's depth rows from bin/chip_report.py's detection_limit, step 50's tables from bin/mei_report.py on a made-up VCF and depth, and step 47's from bin/viral_report.py's build_rows, with made-up counts; steps 02, 40 and 41's tables are written here in their writers' columns. DEMO-001 carries every section, so every card is in the pictures: also the Paraphase card (a PacBio HiFi step, though the invented reads are short) and the couple card, with an invented partner DEMO-002. --sample and --sex write another invented person the same way: the live-site demo (tests/demo/build_live_demo.py) writes DEMO-002, DEMO-001's partner (female), and DEMO-003, their son, so the family page has a trio. A female gets no Y files and her sex checks read two X. A few invented findings make every live-site widget show once in the demo (FINDINGS below): DEMO-002 an FMR1 allele in Stranger's premutation range, a CHIP call and a mosaic loss MoChA reports, DEMO-003 an AnnotSV class 4 deletion. Every sample gets Stranger's thresholds beside ExpansionHunter's counts and step 27's CPIC table, written by bin/pgx_parse.py from the PharmCAT report. The second form reads DIR/DEMO-001/summary.json (step 24 writes it) and fails when a section is not "ok", so a new report card that this script does not feed yet is caught before a picture shows "Not run". Standard library only. tests/demo/retake-screenshots.sh runs both forms. """ import argparse import calendar import contextlib import copy import gzip import io import json import math import os import random import re import shutil import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.dirname(os.path.dirname(HERE)) sys.path.insert(0, os.path.join(REPO, "bin")) import collect_summary # noqa: E402 import pgx_outside_calls # noqa: E402 SAMPLE = "DEMO-001" SEX = "male" PGX_FIXTURES = os.path.join(REPO, "tests", "fixtures", "pgx") PHARMCAT_EXAMPLE = os.path.join(REPO, "tests", "fixtures", "pharmcat", "pharmcat-docs-example.json") # GRCh38 primary chromosomes and their lengths. CHROMS = [ ("chr1", 248956422), ("chr2", 242193529), ("chr3", 198295559), ("chr4", 190214555), ("chr5", 181538259), ("chr6", 170805979), ("chr7", 159345973), ("chr8", 145138636), ("chr9", 138394717), ("chr10", 133797422), ("chr11", 135086622), ("chr12", 133275309), ("chr13", 114364328), ("chr14", 107043718), ("chr15", 101991189), ("chr16", 90338345), ("chr17", 83257441), ("chr18", 80373285), ("chr19", 58617616), ("chr20", 64444167), ("chr21", 46709983), ("chr22", 50818468), ("chrX", 156040895), ("chrY", 57227415), ] AUTOSOMES = [c for c in CHROMS if c[0] not in ("chrX", "chrY")] # Variants per base of each chromosome, relative to an autosome (one X, one Y). DENSITY = {"chrX": 0.45, "chrY": 0.04} def own_chroms(): """The chromosomes the invented person has reads and variants on: a female has no chrY.""" return [c for c in CHROMS if SEX == "male" or c[0] != "chrY"] MEAN_DEPTH = 31.3 # When the invented run started: 2026-10-01 09:00 UTC, the time of the demo's # PharmCAT report too (logs/run_status.tsv, the report's "Latest run" line). RUN_STARTED = calendar.timegm((2026, 10, 1, 9, 0, 0, 0, 0, 0)) # The invented findings of the demo's other people (--sample), one of each live-site widget's marks. FINDINGS = {"DEMO-002": {"fmr1", "chip", "mocha"}, "DEMO-003": {"annotsv"}} # Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), "ATXN3": (44, 60), "AR": (34, 38), "FXN": (33, 66)} def has(finding): return finding in FINDINGS.get(SAMPLE, ()) def put(path, text, gz=False): os.makedirs(os.path.dirname(path), exist_ok=True) if gz: with gzip.open(path, "wt", compresslevel=6) as f: f.write(text) else: with open(path, "w") as f: f.write(text) def vcf_header(extra=(), sample=True): lines = ["##fileformat=VCFv4.2"] + [f"##contig=" for c, n in CHROMS] + list(extra) cols = "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO" lines.append(cols + ("\tFORMAT\t" + SAMPLE if sample else "")) return "\n".join(lines) + "\n" def hla_release(): """HLA_DB_RELEASE's default in scripts/lib/common.sh (3.65.0 -> '3.65.0').""" with open(os.path.join(REPO, "scripts", "lib", "common.sh")) as f: for line in f: m = re.match(r'HLA_DB_RELEASE=\$\{HLA_DB_RELEASE:-([0-9.]+)\}', line.strip()) if m: return m.group(1) raise SystemExit("HLA_DB_RELEASE is not in scripts/lib/common.sh") def image_tag(var): """The tag of an image line of versions.env (PHARMCAT_IMAGE -> 3.4.0).""" with open(os.path.join(REPO, "versions.env")) as f: for line in f: if line.startswith(var + "="): ref = line.split("=", 1)[1].split("#")[0].strip().strip('"') return ref.split("@")[0].rsplit(":", 1)[-1] raise SystemExit(f"{var} is not in versions.env") # --- step 03: the variant calls ------------------------------------------------------ def variants(d, rng, n): """vcf/S.vcf.gz: N records at random positions, about 94% PASS, 85% SNVs.""" chroms = own_chroms() weights = [length * DENSITY.get(c, 1.0) for c, length in chroms] total_w = sum(weights) path = os.path.join(d, "vcf", f"{SAMPLE}.vcf.gz") os.makedirs(os.path.dirname(path), exist_ok=True) head = vcf_header(['##FILTER=', '##FORMAT=', '##FORMAT=', '##FORMAT=']) bases = "ACGT" with gzip.open(path, "wt", compresslevel=1) as f: f.write(head) for (chrom, length), w in zip(chroms, weights): k = round(n * w / total_w) hap = chrom in ("chrX", "chrY") and SEX == "male" out = [] for pos in sorted(rng.sample(range(10001, length - 10000), k)): ref = bases[rng.randrange(4)] r = rng.random() if r < 0.855: alt = bases[(bases.index(ref) + 1 + rng.randrange(3)) % 4] elif r < 0.93: alt = ref + "".join(bases[rng.randrange(4)] for _ in range(1 + rng.randrange(6))) else: alt, ref = ref, ref + "".join(bases[rng.randrange(4)] for _ in range(1 + rng.randrange(6))) dp = max(3, int(rng.gauss(MEAN_DEPTH / (2 if hap else 1), 6))) if rng.random() < 0.942: gt = "1/1" if hap or rng.random() < 0.38 else "0/1" out.append(f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t{rng.randint(20, 66)}.{rng.randrange(10)}\tPASS\t.\t" f"GT:GQ:DP\t{gt}:{rng.randint(20, 60)}:{dp}\n") else: out.append(f"{chrom}\t{pos}\t.\t{ref}\t{alt}\t{rng.randint(0, 9)}.{rng.randrange(10)}\tRefCall\t.\t" f"GT:GQ:DP\t0/0:{rng.randint(5, 30)}:{dp}\n") f.write("".join(out)) # --- step 06: ClinVar screen ----------------------------------------------------------- def clinvar(d, genome_dir): """Three heterozygous hits in genes with invented names (the picture stops above the hits table, and no real gene is tied to an invented person).""" hits = [ ("chr2", 47403191, "C", "T", "DEMOA:90001", "Pathogenic", "criteria_provided,_multiple_submitters,_no_conflicts"), ("chr11", 108259012, "G", "A", "DEMOB:90002", "Likely_pathogenic", "criteria_provided,_single_submitter"), ("chr16", 2087914, "CT", "C", "DEMOC:90003", "Pathogenic", "criteria_provided,_multiple_submitters,_no_conflicts"), ] body = "".join(f"{c}\t{p}\t{90000 + i}\t{r}\t{a}\t50\tPASS\tGENEINFO={g};CLNSIG={s};CLNREVSTAT={rev}\tGT\t0/1\n" for i, (c, p, r, a, g, s, rev) in enumerate(hits)) put(os.path.join(d, "clinvar", f"{SAMPLE}_clinvar_hits.vcf"), vcf_header() + body) # The ClinVar file itself: only the header line the run manifest reads. put(os.path.join(genome_dir, "clinvar", "clinvar.vcf.gz"), "##fileformat=VCFv4.1\n##fileDate=2026-09-28\n##source=ClinVar\n" "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n", gz=True) # --- pharmacogenomics: steps 07, 08, 21, 32 (36 and 27 run for real) ------------------- def pgx(d, genome_dir, rng): """T1K, pypgx, Cyrius and the depth check, then the PharmCAT report that step 07 would write once step 36 passed their consensus on.""" example = json.load(open(PHARMCAT_EXAMPLE)) genes = example["genes"] def called(gene): g = genes[gene]["sourceDiplotypes"][0] return g["label"], "; ".join(g.get("phenotypes") or []) # T1K (step 08) and the CYP2D6 depth check: the invented fixtures. t1k = os.path.join(d, "hla_t1k", f"{SAMPLE}_hla_genotype.tsv") os.makedirs(os.path.dirname(t1k), exist_ok=True) shutil.copy(os.path.join(PGX_FIXTURES, "t1k_genotype.tsv"), t1k) depth = os.path.join(d, "pypgx", f"{SAMPLE}_cyp2d6_depth_check.tsv") os.makedirs(os.path.dirname(depth), exist_ok=True) shutil.copy(os.path.join(PGX_FIXTURES, "depth_check_ok.tsv"), depth) # The HLA database release the manifest prints: one header line of hla.dat, # naming the release setup.sh installs (HLA_DB_RELEASE in scripts/lib/common.sh). put(os.path.join(genome_dir, "t1k_idx", "hlaidx", "hla.dat"), f"ID HLA00001; standard; DNA; HUM; 3503 BP.\nDT 15/07/2025 (Rel. {hla_release()}, Created)\n") # pypgx (step 32) and Cyrius (step 21) agree with PharmCAT's example calls, # CYP2D6 included, so the three CYP2D6 callers agree. cyp, cyp_phen = called("CYP2D6") rows = [] for gene in ("CYP2B6", "CYP2C19", "CYP2C9", "CYP2D6", "CYP3A5", "NUDT15", "SLCO1B1", "TPMT", "UGT1A1"): dip, phen = called(gene) rows.append(f"{gene}\t{dip}\t{phen}\t{'Normal' if gene == 'CYP2D6' else 'N/A'}\t{'BAM' if gene == 'CYP2D6' else 'VCF'}\n") put(os.path.join(d, "pypgx", f"{SAMPLE}_pypgx_summary.tsv"), "Gene\tDiplotype\tPhenotype\tCNV_call\tSource\n" + "".join(rows)) put(os.path.join(d, "cyrius", f"{SAMPLE}_cyp2d6.tsv"), f"Sample\tGenotype\tFilter\n{SAMPLE}\t{cyp}\tPASS\n") # What step 36 will pass to PharmCAT, from its own code. t1k_calls = pgx_outside_calls.read_t1k(t1k) outside = {} for gene in pgx_outside_calls.HLA_GENES: _, dip = pgx_outside_calls.hla_row(gene, t1k_calls) if dip: outside[gene] = dip _, dip = pgx_outside_calls.cyp2d6_row(os.path.join(d, "pypgx", f"{SAMPLE}_pypgx_summary.tsv"), os.path.join(d, "cyrius", f"{SAMPLE}_cyp2d6.tsv"), depth) if dip: outside["CYP2D6"] = dip # The PharmCAT report (step 07, run after step 36): PharmCAT's example, # with DEMO-001's outside calls. HLA phenotypes as PharmCAT writes them. report = copy.deepcopy(example) report.pop("_fixture_note", None) report["title"] = SAMPLE report["timestamp"] = "2026-10-01T09:00:00.000Z" report["pharmcatVersion"] = image_tag("PHARMCAT_IMAGE") hla_tests = {"HLA-A": ["*31:01"], "HLA-B": ["*15:02", "*57:01", "*58:01"]} changed = {} for gene, dip in outside.items(): alleles = dip.split("/") if gene in hla_tests: phen = [f"{t} {'positive' if t in alleles else 'negative'}" for t in hla_tests[gene]] else: phen = [cyp_phen] if dip == cyp else ["Indeterminate"] if dip != called(gene)[0]: changed[gene] = dip report["genes"][gene]["callSource"] = "OUTSIDE" report["genes"][gene]["sourceDiplotypes"] = [{ "gene": gene, "label": dip, "phenotypes": phen, "activityScore": None, "outsidePhenotype": False, "allele1": {"gene": gene, "name": alleles[0], "function": None}, "allele2": {"gene": gene, "name": alleles[1], "function": None}}] # MT-RNR1 is an outside call in PharmCAT's example; this pipeline passes # none for it, so it is read from the VCF here, as the reference allele. mt = report["genes"]["MT-RNR1"] mt["callSource"] = "MATCHER" mt["sourceDiplotypes"] = [{"gene": "MT-RNR1", "label": "Reference", "phenotypes": ["normal risk of aminoglycoside-induced hearing loss"], "activityScore": None, "outsidePhenotype": False, "allele1": {"gene": "MT-RNR1", "name": "Reference", "function": None}, "allele2": None}] changed["MT-RNR1"] = "Reference" prune_annotations(report, changed) put(os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), json.dumps(report, indent=1) + "\n") # Step 27's tables from that report, by the parser step 27 runs (bin/pgx_parse.py). import pgx_parse with contextlib.redirect_stdout(io.StringIO()): if pgx_parse.main(["cpic-report", "--sample", SAMPLE, "--report", os.path.join(d, "pharmcat", f"{SAMPLE}.report.json"), "--outdir", os.path.join(d, "cpic")]): raise SystemExit("pgx_parse cpic-report failed") return outside def prune_annotations(report, changed): """Drop the example's drug annotations written for a diplotype of a gene whose call changed: PharmCAT lists only those that fit the sample.""" for src, by_drug in list(report.get("drugs", {}).items()): for name, rep in list(by_drug.items()): for gl in rep.get("guidelines") or []: keep = [] for ann in gl.get("annotations") or []: labels = {(dp.get("gene"), dp.get("label")) for gt in ann.get("genotypes") or [] for dp in (gt or {}).get("diplotypes") or [] if isinstance(dp, dict)} stale = any(g in changed and lab != changed[g] for g, lab in labels) stale = stale or any(g in changed and not any(x[0] == g for x in labels) for g in (ann.get("phenotypes") or {})) if not stale: keep.append(ann) gl["annotations"] = keep rep["guidelines"] = [gl for gl in rep.get("guidelines") or [] if gl.get("annotations")] if not rep["guidelines"]: del by_drug[name] if not by_drug: del report["drugs"][src] # --- step 25: polygenic scores through pgsc_calc's files ------------------------------- def prs(d, genome_dir, rng): labels = collect_summary.read_labels(os.path.join(REPO, "assets", "pgs_scores.tsv")) harmonised = os.path.join(genome_dir, "prs_scores") for pid, trait in labels.items(): n = rng.randint(250, 4000) rows = [] for _ in range(n): c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] ea, oa = rng.sample("ACGT", 2) rows.append(f"{c[3:]}\t{rng.randint(1, length)}\t{ea}\t{oa}\t{rng.gauss(0, 0.05):.5f}\t{c[3:]}\t{rng.randint(1, length)}\n") put(os.path.join(harmonised, f"{pid}.txt.gz"), f"#format_version=2.0\n#pgs_id={pid}\n#pgs_name={pid}\n#trait_reported={trait}\n#variants_number={n}\n" "#HmPOS_build=GRCh38\nchr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\thm_chr\thm_pos\n" + "".join(rows), gz=True) work = os.path.join(d, "prs", "pgsc_calc") scores = os.path.join(work, "scores") if collect_summary.prs_format_main(["--scores", harmonised, "--labels", os.path.join(REPO, "assets", "pgs_scores.tsv"), "--out", scores, "--alleles", os.path.join(work, "score_alleles.tsv")]): raise SystemExit("prs-format failed") # pgsc_calc's output for sampleset "sample" (step 25's name) with the 1000 Genomes panel. res = os.path.join(work, "results", "sample") summ, score = ["dataset,accession,ambiguous,is_multiallelic,match_flipped,duplicate_best_match," "duplicate_ID,match_IDs,match_status,count,score_pass,match_rate"], [] for pid in sorted(labels): with gzip.open(os.path.join(scores, f"{pid}.txt.gz"), "rt") as f: total = sum(1 for line in f if not line.startswith("#")) - 1 m = round(total * rng.uniform(0.86, 0.99)) summ.append(f"sample,{pid},false,false,false,false,false,NA,matched,{m},true,{m / total:.2f}") summ.append(f"sample,{pid},false,false,false,false,false,NA,unmatched,{total - m},true,{m / total:.2f}") score.append(f"sample\t{SAMPLE}\t{SAMPLE}\t{pid}\t{rng.gauss(0, 0.4):.4f}\t{rng.gauss(0, 1):.3f}\t" f"{rng.uniform(18, 82):.2f}") put(os.path.join(res, "match", "sample_summary.csv"), "\n".join(summ) + "\n") put(os.path.join(res, "score", "sample_pgs.txt.gz"), "sampleset\tFID\tIID\tPGS\tSUM\tZ_MostSimilarPop\tpercentile_MostSimilarPop\n" + "\n".join(score) + "\n", gz=True) pcs = "\t".join(f"{rng.gauss(0, 0.01):.4f}" for _ in range(10)) put(os.path.join(res, "score", "sample_popsimilarity.txt.gz"), "sampleset\tFID\tIID\t" + "\t".join(f"PC{i}" for i in range(1, 11)) + "\tRF_P_AFR\tRF_P_AMR\tRF_P_EAS\tRF_P_EUR\tRF_P_SAS\tMostSimilarPop\tMostSimilarPop_LowConfidence\tREFERENCE\n" f"sample\t{SAMPLE}\t{SAMPLE}\t{pcs}\t0.00\t0.02\t0.00\t0.97\t0.01\tEUR\tFalse\tFalse\n", gz=True) os.makedirs(os.path.join(d, "ancestry"), exist_ok=True) if collect_summary.prs_table_main(["--sample", SAMPLE, "--results", os.path.join(work, "results"), "--sampleset", "sample", "--scores", scores, "--input-kind", "gvcf", "--panel", image_tag_panel(), "--ancestry-out", os.path.join(d, "ancestry", f"{SAMPLE}_ancestry.tsv"), "--out", os.path.join(d, "prs", f"{SAMPLE}_prs_summary.tsv")]): raise SystemExit("prs-table failed") def image_tag_panel(): with open(os.path.join(REPO, "versions.env")) as f: for line in f: if line.startswith("PGSC_PANEL="): return line.split("=", 1)[1].split("#")[0].strip().strip('"') return "pgsc_1000G_v1" # --- steps 16, 16b, 33, 01b and samtools: QC ------------------------------------------ def coverage(d, rng): """mosdepth (step 16b) and indexcov (step 16).""" m = os.path.join(d, "mosdepth") summary = ["chrom\tlength\tbases\tmean\tmin\tmax"] dist = [] tot_len = tot_bases = 0 def cumulative(mean, sd, gap): # Share of bases with at least each depth: a normal curve, plus the # gaps of the reference (no reads) at depth 0. out = [] for dep in range(0, int(mean * 2.6) + 1): p = (1 - gap) * 0.5 * math.erfc((dep - 0.5 - mean) / (sd * math.sqrt(2))) if dep else 1.0 if p < 0.005 and dep: break out.append((dep, p)) return out curves = {} for chrom, length in own_chroms(): half = chrom in ("chrX", "chrY") and SEX == "male" mean = MEAN_DEPTH / (2 if half else 1) * rng.uniform(0.97, 1.03) gap = rng.uniform(0.01, 0.03) if chrom != "chrY" else 0.55 bases = int(length * mean * (1 - gap)) summary.append(f"{chrom}\t{length}\t{bases}\t{bases / length:.2f}\t0\t{int(mean * 9)}") tot_len += length tot_bases += bases curves[chrom] = cumulative(mean, mean * 0.24, gap) summary.append(f"total\t{tot_len}\t{tot_bases}\t{tot_bases / tot_len:.2f}\t0\t{int(MEAN_DEPTH * 9)}") for chrom, _ in own_chroms(): dist += [f"{chrom}\t{dep}\t{p:.2f}" for dep, p in curves[chrom]] dist += [f"total\t{dep}\t{p:.2f}" for dep, p in (cumulative(MEAN_DEPTH, MEAN_DEPTH * 0.25, 0.02))] put(os.path.join(m, f"{SAMPLE}.mosdepth.summary.txt"), "\n".join(summary) + "\n") put(os.path.join(m, f"{SAMPLE}.mosdepth.global.dist.txt"), "\n".join(dist) + "\n") i = os.path.join(d, "indexcov") put(os.path.join(i, "indexcov-indexcov.ped"), "#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\tCNchrX\tCNchrY\tbins.out\tbins.lo\tbins.hi\t" "bins.in\tslope\tp.out\tPC1\tPC2\tPC3\tPC4\tPC5\n" f"unknown\t{SAMPLE}\t-9\t-9\t{1 if SEX == 'male' else 2}\t-9\t{'1.01' if SEX == 'male' else '1.99'}\t" f"{'0.98' if SEX == 'male' else '0.01'}\t312\t205\t107\t174820\t0.993\t0.0018\t" "0\t0\t0\t0\t0\n") roc = ["#chrom\tcov\t" + SAMPLE] for chrom, _ in own_chroms(): half = chrom in ("chrX", "chrY") and SEX == "male" centre = 0.5 if half else 1.0 for k in range(0, 41): cov = k * 0.05 frac = 0.5 * math.erfc((cov - centre) / (0.12 * math.sqrt(2))) roc.append(f"{chrom}\t{cov:.2f}\t{min(1.0, frac):.3f}") put(os.path.join(i, "indexcov-indexcov.roc"), "\n".join(roc) + "\n") def sample_qc(d): """somalier and VerifyBamID2 files, then step 33's own table writer. sex is the sex somalier infers from the reads. original_pedigree_sex stays -9 (unknown), as in a real run: step 33 gives somalier no pedigree file, so MultiQC's somalier "Sex" column (that pedigree sex) reads -9.""" q = os.path.join(d, "qc") sex = 1 if SEX == "male" else 2 put(os.path.join(q, "somalier", f"{SAMPLE}.samples.tsv"), "#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\toriginal_pedigree_sex\tgt_depth_mean\t" "gt_depth_sd\tdepth_mean\tdepth_sd\tab_mean\tab_std\tn_hom_ref\tn_het\tn_hom_alt\tn_unknown\tp_middling_ab\t" "X_depth_mean\tX_n\tX_hom_ref\tX_het\tX_hom_alt\tY_depth_mean\tY_n\n" f"{SAMPLE}\t{SAMPLE}\t-9\t-9\t{sex}\t-9\t-9\t31.4\t7.2\t31.4\t7.2\t0.41\t0.08\t9817\t6904\t4383\t196\t0.01\t" + ("15.6\t612\t352\t4\t256\t14.9\t41\n" if SEX == "male" else "31.2\t612\t300\t210\t102\t0.1\t0\n")) put(os.path.join(q, "somalier", f"{SAMPLE}.pairs.tsv"), "#sample_a\tsample_b\trelatedness\tibs0\tibs2\thom_concordance\thets_a\thets_b\thets_ab\tshared_hets\t" "hom_alts_a\thom_alts_b\tshared_hom_alts\tn\tx_ibs0\tx_ibs2\texpected_relatedness\n") put(os.path.join(q, "verifybamid2", f"{SAMPLE}.selfSM"), "#SEQ_ID\tRG\tCHIP_ID\t#SNPS\t#READS\tAVG_DP\tFREEMIX\tFREELK1\tFREELK0\tFREE_RH\tFREE_RA\tCHIPMIX\tCHIPLK1\t" "CHIPLK0\tCHIP_RH\tCHIP_RA\tDPREF\tRDPHET\tRDPALT\n" f"{SAMPLE}\tNA\tNA\t99846\t3087112\t30.92\t0.00412\t-1418823.2\t-1419102.7\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\n") if collect_summary.sample_qc_main(["--sample", SAMPLE, "--somalier-samples", os.path.join(q, "somalier", f"{SAMPLE}.samples.tsv"), "--somalier-pairs", os.path.join(q, "somalier", f"{SAMPLE}.pairs.tsv"), "--selfsm", os.path.join(q, "verifybamid2", f"{SAMPLE}.selfSM"), "--declared-sex", SEX, "--marker-check", "passed", "--out", os.path.join(q, f"{SAMPLE}_sample_qc.tsv")]): raise SystemExit("sample-qc failed") def reads(d, rng): """fastp's JSON (step 01b) and samtools flagstat (step 28 writes it from the BAM).""" pairs = 412_906_118 before = pairs * 2 passed = int(before * 0.9891) low_q, too_short, n_reads = int(before * 0.0081), int(before * 0.0026), before - passed - int(before * 0.0081) - int(before * 0.0026) length = 151 def curves(after): q = [round(min(37.4, 31.5 + 6 * (1 - math.exp(-i / 4))) - (i / length) ** 3 * (2.2 if after else 3.5) + rng.uniform(-0.15, 0.15), 2) for i in range(length)] base = {b: [round(v + rng.uniform(-0.004, 0.004), 4) for v in [p] * length] for b, p in (("A", 0.295), ("T", 0.296), ("C", 0.204), ("G", 0.205))} content = dict(base, N=[0.0002] * length, GC=[round(base["C"][i] + base["G"][i], 4) for i in range(length)]) return {"total_reads": (passed if after else before) // 2, "total_bases": (passed if after else before) // 2 * length, "q20_bases": int((passed if after else before) // 2 * length * 0.968), "q30_bases": int((passed if after else before) // 2 * length * 0.921), "total_cycles": length, "quality_curves": dict({b: q for b in "ATCG"}, mean=q), "content_curves": content} def stats(n, gc): return {"total_reads": n, "total_bases": n * length, "q20_bases": int(n * length * 0.968), "q30_bases": int(n * length * 0.921), "q20_rate": 0.968, "q30_rate": 0.921, "read1_mean_length": length, "read2_mean_length": length, "gc_content": gc} hist = [0] * 1000 for i in range(1000): hist[i] = int(4.2e6 * math.exp(-((i - 382) / 118) ** 2 / 2)) fastp = { "summary": {"fastp_version": "1.0.1", "sequencing": "paired end (151 cycles + 151 cycles)", "before_filtering": stats(before, 0.4105), "after_filtering": stats(passed, 0.4098)}, "filtering_result": {"passed_filter_reads": passed, "low_quality_reads": low_q, "too_many_N_reads": n_reads, "too_short_reads": too_short, "too_long_reads": 0}, "duplication": {"rate": 0.0712}, "insert_size": {"peak": 382, "unknown": 5102117, "histogram": hist}, "adapter_cutting": {"adapter_trimmed_reads": int(before * 0.074), "adapter_trimmed_bases": int(before * 1.9), "read1_adapter_sequence": "AGATCGGAAGAGCACACGTCTGAACTCCAGTCA", "read2_adapter_sequence": "AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT"}, "read1_before_filtering": curves(False), "read2_before_filtering": curves(False), "read1_after_filtering": curves(True), "read2_after_filtering": curves(True), "command": f"fastp -i /genome/{SAMPLE}/fastq/{SAMPLE}_R1.fastq.gz -I /genome/{SAMPLE}/fastq/{SAMPLE}_R2.fastq.gz " f"-o /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_R1.fastq.gz -O /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_R2.fastq.gz " f"-j /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_fastp.json -h /genome/{SAMPLE}/fastq_trimmed.part/{SAMPLE}_fastp.html", } put(os.path.join(d, "fastq_trimmed", f"{SAMPLE}_fastp.json"), json.dumps(fastp, indent=1) + "\n") primary = passed supp = int(primary * 0.0034) total = primary + supp mapped = int(primary * 0.9962) + supp dups = int(primary * 0.0712) paired = primary proper = int(paired * 0.981) single = int(paired * 0.0019) both = int(paired * 0.9943) diff = int(paired * 0.0061) flag = [ f"{total} + 0 in total (QC-passed reads + QC-failed reads)", f"{primary} + 0 primary", "0 + 0 secondary", f"{supp} + 0 supplementary", f"{dups} + 0 duplicates", f"{dups} + 0 primary duplicates", f"{mapped} + 0 mapped ({100 * mapped / total:.2f}% : N/A)", f"{mapped - supp} + 0 primary mapped ({100 * (mapped - supp) / primary:.2f}% : N/A)", f"{paired} + 0 paired in sequencing", f"{paired // 2} + 0 read1", f"{paired // 2} + 0 read2", f"{proper} + 0 properly paired ({100 * proper / paired:.2f}% : N/A)", f"{both} + 0 with itself and mate mapped", f"{single} + 0 singletons ({100 * single / paired:.2f}% : N/A)", f"{diff} + 0 with mate mapped to a different chr", f"{int(diff * 0.62)} + 0 with mate mapped to a different chr (mapQ>=5)", ] put(os.path.join(d, "aligned", f"{SAMPLE}_flagstat.txt"), "\n".join(flag) + "\n") # --- structural variants: steps 04, 18, 19, 22 ------------------------------------------ def sv_records(rng, n, caller, pass_rate): out = [] for i in range(n): c, length = CHROMS[rng.randrange(len(CHROMS) - 1)] t = rng.choices(["DEL", "DUP", "INV", "INS"], weights=[62, 14, 6, 18])[0] pos = rng.randint(20000, length - 2_000_000) size = int(math.exp(rng.uniform(math.log(60), math.log(400000)))) end = pos + (1 if t == "INS" else size) svlen = -size if t == "DEL" else size filt = "PASS" if rng.random() < pass_rate else rng.choice(["MinQUAL", "MinGQ", "LowQual"]) gt = "1/1" if rng.random() < 0.2 else "0/1" out.append((c, pos, f"{caller}{t}:{i}", t, end, svlen, filt, gt, rng.randint(20, 999))) out.sort(key=lambda r: ([x[0] for x in CHROMS].index(r[0]), r[1])) return out def structural(d, rng): info = ['##INFO=', '##INFO=', '##INFO='] def write(path, recs, extra=()): body = "".join(f"{c}\t{p}\t{i}\tN\t<{t}>\t{q}\t{f}\tEND={e};SVTYPE={t};SVLEN={l}{x}\tGT\t{g}\n" for (c, p, i, t, e, l, f, g, q), x in zip(recs, extra or [""] * len(recs))) put(path, vcf_header(info + ['##FORMAT=']) + body, gz=True) write(os.path.join(d, "manta", "results", "variants", "diploidSV.vcf.gz"), sv_records(rng, rng.randint(7600, 8000), "Manta", 0.82)) write(os.path.join(d, "delly", f"{SAMPLE}_sv.vcf.gz"), sv_records(rng, rng.randint(5900, 6300), "Delly", 0.86)) # SURVIVOR's merge of Manta, Delly and CNVpytor (step 22): SUPP and SUPP_VEC. merged = sv_records(rng, rng.randint(2700, 3100), "SURVIVOR", 1.0) vecs = [rng.choices(["110", "101", "011", "111"], weights=[70, 9, 6, 15])[0] for _ in merged] write(os.path.join(d, "sv_merged", f"{SAMPLE}_sv_consensus.vcf.gz"), merged, [f";SUPP={v.count('1')};SUPP_VEC={v}" for v in vecs]) lines = [] for _ in range(rng.randint(1100, 1250)): c, length = CHROMS[rng.randrange(len(CHROMS) - 1)] t = "deletion" if rng.random() < 0.83 else "duplication" size = rng.randrange(1000, 400000, 100) start = rng.randrange(10001, length - size, 100) cn = rng.uniform(0.02, 0.6) if t == "deletion" else rng.uniform(1.3, 2.1) p = 10 ** rng.uniform(-40, -0.5) lines.append(f"{t}\t{c}:{start}-{start + size - 1}\t{size}\t{cn:.4f}\t{p:.4e}\t{p * 3:.4e}\t{min(1, p * 5):.4e}\t" f"{min(1, p * 9):.4e}\t{rng.uniform(0, 0.5):.4f}\t{rng.uniform(0, 0.1):.4f}\t{rng.randint(0, 9)}") put(os.path.join(d, "cnvpytor", f"{SAMPLE}_cnvs.txt"), "\n".join(lines) + "\n") # Step 05's class 4 and 5 SVs, checked (bin/annotsv_postfilter.py's # columns): none, as in most genomes. import annotsv_postfilter cols = annotsv_postfilter.KEEP + annotsv_postfilter.ADDED rows = [] if has("annotsv"): # One class 4 deletion over an invented gene, from Manta, kept by step 05's rules. rows.append({"AnnotSV_ID": "chr16_2084000_2096000_DEL_1", "ID": "MantaDEL:1", "SV_chrom": "chr16", "SV_start": "2084000", "SV_end": "2096000", "SV_length": "-12000", "SV_type": "DEL", "Gene_name": "G0533", "ACMG_class": "4", "AnnotSV_ranking_score": "0.92", "Caller": "Manta", "Exon_edge": "no", "Exon_edge_detail": ".", "Population_AF": "", "Population_source": "", "Common": "no", "AnnotSV_class": "4", "Depth_support": ".", "Rule": "", "Rule_detail": ""}) put(os.path.join(d, "annotsv", f"{SAMPLE}_sv_class4_5_checked.tsv"), "\t".join(cols) + "\n" + "".join("\t".join(r[k] for k in cols) + "\n" for r in rows)) # --- repeats, telomeres, ROH, mitochondria, Y: steps 09, 10, 11, 12, 20, 37 -------------- # ExpansionHunter's key loci with the normal range each repeat count is drawn from. EH_RANGES = [("HTT", "chr4", 3074877, "CAG", 10, 26), ("FMR1", "chrX", 147912051, "CGG", 20, 40), ("C9ORF72", "chr9", 27573529, "GGCCCC", 2, 10), ("ATXN1", "chr6", 16327636, "TGC", 25, 35), ("DMPK", "chr19", 45770205, "CAG", 5, 30), ("ATXN3", "chr14", 92071011, "GCT", 14, 30), ("AR", "chrX", 67545317, "GCA", 17, 26), ("FXN", "chr9", 69037287, "GAA", 6, 30)] def repeats_and_more(d, rng): eh, stranger = [], [] for rep, c, pos, unit, lo, hi in EH_RANGES: hap = c == "chrX" and SEX == "male" a = sorted(rng.randint(lo, hi) for _ in range(1 if hap else 2)) if rep == "FMR1" and has("fmr1") and not hap: a = [a[0], 62] # an allele in Stranger's premutation range (above 54, under 200) ref = (lo + hi) // 2 gt = "1" if hap else ("1/2" if a[0] != a[1] else "1/1") cn = "/".join(map(str, a)) alts = ",".join(f"" for x in dict.fromkeys(a)) info = f"END={pos + ref * len(unit)};REF={ref};RL={ref * len(unit)};RU={unit};VARID={rep};REPID={rep}" lc = f"{rng.uniform(28, 34):.6f}" eh.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info}\tGT:SO:REPCN:LC\t{gt}:SPANNING:{cn}:{lc}\n") # Step 9b's Stranger copy: the same record with the catalog's thresholds, the worst status of # the alleles, and ExpansionHunter's 95% interval of each (REPCI), a few repeats wide past 50. nmax, pmin = STR_THRESHOLDS[rep] worst = ("full_mutation" if any(x >= pmin for x in a) else "pre_mutation" if any(x > nmax for x in a) else "normal") ci = "/".join(f"{x}-{x}" if x < 50 else f"{x - 4}-{x + 5}" for x in a) stranger.append(f"{c}\t{pos}\t.\t{unit[0]}\t{alts}\t.\tPASS\t{info};STR_STATUS={worst};STR_NORMAL_MAX={nmax};" f"STR_PATHOLOGIC_MIN={pmin}\tGT:SO:REPCN:REPCI:LC\t{gt}:SPANNING:{cn}:{ci}:{lc}\n") head = vcf_header(['##ALT=']) put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh.vcf"), head + "".join(eh)) put(os.path.join(d, "expansion_hunter", f"{SAMPLE}_eh_stranger.vcf"), head + "".join(stranger)) put(os.path.join(d, "telomere", SAMPLE, f"{SAMPLE}_summary.tsv"), "PID\tsample\ttotal_reads\tread_lengths\trepeat_threshold_set\trepeat_threshold_used\tintratelomeric_reads\t" "gc_bins_for_correction\ttotal_reads_with_tel_gc\ttel_content\n" f"{SAMPLE}\ttumor\t825812236\t151\tn=7 per 100 bp\t10\t61873\t48-52\t185204533\t{rng.uniform(260, 480):.2f}\n") segs, total = [], 0 while total < 44e6: c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] size = int(rng.uniform(0.5e6, 3.9e6)) start = rng.randint(1_000_000, length - size - 1_000_000) segs.append((AUTOSOMES.index((c, length)), c, start, start + size, size)) total += size segs.sort() put(os.path.join(d, "vcf", f"{SAMPLE}_roh.txt"), "# RG, Regions [2]Sample\t[3]Chromosome\t[4]Start\t[5]End\t[6]Length (bp)\t[7]Number of markers\t[8]Quality\n" + "".join(f"RG\t{SAMPLE}\t{c}\t{s}\t{e}\t{n}\t{n // 1400}\t{rng.uniform(40, 90):.1f}\n" for _, c, s, e, n in segs)) mito = [] het = {11, 23} for i, pos in enumerate(sorted(rng.sample(range(60, 16500), 41))): ref = "ACGT"[rng.randrange(4)] alt = "ACGT"[("ACGT".index(ref) + 2) % 4] if i in het: af = rng.uniform(0.11, 0.38) else: af = rng.uniform(0.985, 1.0) if i < 37 else rng.uniform(0.01, 0.03) filt = "PASS" if i < 37 else "weak_evidence" mito.append(f"chrM\t{pos}\t.\t{ref}\t{alt}\t.\t{filt}\t.\tGT:AF:DP\t0/1:{af:.3f}:{rng.randint(2400, 3300)}\n") put(os.path.join(d, "mito", f"{SAMPLE}_chrM_filtered.vcf.gz"), "##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + SAMPLE + "\n" + "".join(mito), gz=True) put(os.path.join(d, "mito", f"{SAMPLE}_haplogroup.txt"), '"SampleID"\t"Haplogroup"\t"Rank"\t"Quality"\t"Range"\n' f'"{SAMPLE}"\t"H1c3"\t"1"\t"0.9312"\t"1-16569;"\n') # Read once wave 11's report has the haplocheck line and the Y card. put(os.path.join(d, "mito", f"{SAMPLE}_haplocheck.txt"), '"Sample"\t"Contamination Status"\t"Contamination Level"\t"Distance"\t"Sample Coverage"\n' f'"{SAMPLE}"\t"NO"\t"ND"\t"0"\t"2861"\n') if SEX != "male": return # no Y chromosome: step 37 has nothing to place put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_haplogroup.txt"), "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" f"{SAMPLE}_sorted\tI-M253\tM253\t4021876\t1187\t0.982\t1.0\t0.991\t0.982\n") # Step 37's placement on the current tree (bin/y_tree_place.py's columns), read # by the report's y_tree section; the same I-M253 as above, five levels down. # As the real writer does: '.' for an empty SNP list, and its default depth of 10. put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_tree.txt"), "Sample_name\tHg\tLevel\tPath_derived\tPath_ancestral\tPath_no_call\tAncestral_in_terminal\t" "SNPs_called\tMin_depth\tTree\tPositions\tOutvoted_ancestral\n" f"{SAMPLE}\tI-M253\t5\t41\t0\t3\t.\t1187\t10\tYFull YTree v14.06.0\tYleaf hg38 position tables\t.\n") put(os.path.join(d, "y_haplogroup", f"{SAMPLE}_y_tree_path.tsv"), "Level\tBranch\tSNPs\tOn_tables\tDerived\tAncestral\tNo_call\tAncestral_SNPs\n" "1\tCT\t12\t12\t11\t0\t1\t.\n" "2\tF\t9\t9\t9\t0\t0\t.\n" "3\tIJ\t8\t8\t7\t0\t1\t.\n" "4\tI\t7\t7\t7\t0\t0\t.\n" "5\tI-M253\t8\t8\t7\t0\t1\t.\n") # --- steps 17, 23, 31 ---------------------------------------------------------------- def clinical(d, rng): def gene(): return f"G{rng.randint(1, 900):04d}" impacts = ["HIGH"] * 41 + ["MODERATE"] * 233 + ["LOW"] * 44 rows = [] for imp in impacts: c, length = AUTOSOMES[rng.randrange(len(AUTOSOMES))] rows.append(f"{c}\t{rng.randint(1, length)}\tC\tT\t0/1\t{imp}\t{gene()}\tmissense_variant\t0.0004\t24.1\t0.41\t.\n") put(os.path.join(d, "clinical", f"{SAMPLE}_clinical_summary.tsv"), "CHROM\tPOS\tREF\tALT\tGT\tIMPACT\tGENE\tConsequence\tMAX_AF\tCADD_PHRED\tREVEL\tAM_CLASS\n" + "".join(rows)) sl = [f"chr{rng.randint(1, 22)}\t{rng.randint(1, 90_000_000)}\tA\tG\t{rng.choice(['HIGH', 'MODERATE'])}\t{gene()}\n" for _ in range(57)] put(os.path.join(d, "slivar", f"{SAMPLE}_slivar_summary.tsv"), "CHROM\tPOS\tREF\tALT\tIMPACT\tGENE\n" + "".join(sl)) ch = [] for _ in range(3): g = gene() ch += [f"chr{rng.randint(1, 22)}\t{rng.randint(1, 90_000_000)}\tA\tG\tMODERATE\t{g}\n" for _ in range(2)] put(os.path.join(d, "slivar", f"{SAMPLE}_compound_hets.tsv"), "CHROM\tPOS\tREF\tALT\tIMPACT\tGENE\n" + "".join(ch)) counts = {"Benign": 289, "Likely_Benign": 112, "VUS": 38} cls = "".join(f"{SAMPLE}\tchr{rng.randint(1, 22)}:g.{rng.randint(1, 90_000_000)}C>T\t{k}\t{k}\n" for k, n in counts.items() for _ in range(n)) put(os.path.join(d, "cpsr", f"{SAMPLE}.cpsr.grch38.classification.tsv.gz"), "SAMPLE_ID\tGENOMIC_CHANGE\tCPSR_CLASSIFICATION\tCLASSIFICATION\n" + cls, gz=True) put(os.path.join(d, "cpsr", f"{SAMPLE}.cpsr.grch38.html"), f"CPSR {SAMPLE}Invented sample.\n") # --- steps 02, 33 (mosaic), 26b, 35b, 39, 40, 41 and 42 --------------------------------- # The sections added since wave 18. Each table is written by the writer its # step uses (bin/*.py) from made-up inputs where that writer is a function, # else in the writer's own columns. def read_count(d): """qc/S_read_count.tsv as read_count_check (scripts/lib/common.sh) writes it: the BAM holds every read fastp kept, so the check is ok.""" with open(os.path.join(d, "fastq_trimmed", f"{SAMPLE}_fastp.json")) as f: kept = json.load(f)["filtering_result"]["passed_filter_reads"] r1 = r2 = kept // 2 src = (f"fastq_trimmed/{SAMPLE}_R1.fastq.gz {r1} records + fastq_trimmed/{SAMPLE}_R2.fastq.gz {r2} records") put(os.path.join(d, "qc", f"{SAMPLE}_read_count.tsv"), f"key\tvalue\ninput_reads\t{r1 + r2}\ninput_source\t{src}\nprimary_records\t{r1 + r2}\n" "missing_reads\t0\nread_count_check\tok\n") # Genes the demo leaves below step 16b's share: four that another step owns and # four with no other reading, so both lines of the card have names in them. DEMO_NOT_READ = {"SMN1", "SMN2", "NCF1", "PMS2", "NBPF1", "ANKRD36C", "DUX4", "MUC12"} def callability(d, rng): """Step 16b's gene table (mosdepth --thresholds 10 over the genes of assets/callability_genes.tsv: chrom, start, end, gene, bases at 10x), written from the repository's catalog, so it always matches it. The ALT-only genes get no row, as in the real step.""" rows = ["#chrom\tstart\tend\tregion\t10X"] for r in collect_summary.asset_rows("callability_genes.tsv"): if r["start"] == ".": continue start, end = int(r["start"]), int(r["end"]) share = rng.uniform(0.35, 0.8) if r["gene"] in DEMO_NOT_READ else rng.uniform(0.93, 1.0) rows.append(f"{r['chrom']}\t{start}\t{end}\t{r['gene']}\t{int((end - start) * share)}") put(os.path.join(d, "mosdepth", f"{SAMPLE}_callability.bed.gz"), "\n".join(rows) + "\n", gz=True) def mosaic(d, rng): """mosdepth's 500 bp windows (step 16b's regions.bed.gz), cut down to three 200 kb stretches of each autosome and the single-copy part of chrY, then bin/mosaic_sex_loss.py itself adds its rows to step 33's table. One Y in every cell: the ratio sits near 0.5 and no loss is detected.""" import mosaic_sex_loss path = os.path.join(d, "mosdepth", f"{SAMPLE}.regions.bed.gz") lines = [] for chrom, length in AUTOSOMES: mean = MEAN_DEPTH * rng.uniform(0.98, 1.02) for k in range(3): start = (length // 4) * (k + 1) // 500 * 500 lines += [f"{chrom}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(mean, mean * 0.12)):.2f}" for s in range(start, start + 200_000, 500)] # A male's single-copy chrY at half the depth; a female's chrX (two copies) at the full depth. sx = "Y" if SEX == "male" else "X" ymean = MEAN_DEPTH / (2 if SEX == "male" else 1) * rng.uniform(0.98, 1.0) for start, end in mosaic_sex_loss.SINGLE_COPY[sx][:10]: s0 = (start // 500 + 1) * 500 lines += [f"chr{sx}\t{s}\t{s + 500}\t{max(0.0, rng.gauss(ymean, ymean * 0.14)):.2f}" for s in range(s0, min(end, s0 + 1_000_000) - 500, 500)] put(path, "\n".join(lines) + "\n", gz=True) if mosaic_sex_loss.main(["--table", os.path.join(d, "qc", f"{SAMPLE}_sample_qc.tsv"), "--regions", path]): raise SystemExit("mosaic_sex_loss failed") def ancestry_fine(d, rng): """Step 26b's table (ADMIXTURE fractions with their bootstrap range, then the painted shares) and its painting, drawn by bin/ancestry_fine_report.py. Mostly West Eurasian, as step 25's EUR placement above.""" import ancestry_fine_report as afr continent = {"WEA": 0.942, "CSA": 0.031, "AFR": 0.012, "EAS": 0.008, "AMR": 0.005, "OCE": 0.002} region = {"NWE": 0.463, "IBE": 0.274, "ITA": 0.118, "NEE": 0.052, "SAR": 0.009, "CAU": 0.011, "LEV": 0.008, "BDN": 0.002, "NAF": 0.005, "CSA": 0.031, "WAF": 0.007, "EAF": 0.003, "CAF": 0.001, "SAF": 0.001, "EAS": 0.006, "SIB": 0.002, "AMR": 0.005, "OCE": 0.002} table = [] for level, fr in (("continent", continent), ("region", region)): for code, f in sorted(fr.items(), key=lambda kv: -kv[1]): se = max(0.002, math.sqrt(f * (1 - f)) * rng.uniform(0.04, 0.08)) table.append([level, code, afr.label(level, code), f"{f:.4f}", f"{se:.4f}", f"{max(0.0, f - 1.96 * se):.4f}", f"{min(1.0, f + 1.96 * se):.4f}"]) # The painting: each haplotype of chr1-22 in segments of 8 to 60 Mb, West # Eurasian but for a few short ones. rows = [] for i, n in enumerate(afr.CHROM_LEN): for hap in (1, 2): pos = rng.randint(200_000, 1_500_000) while pos < n - 2_000_000: end = min(n - rng.randint(200_000, 1_500_000), pos + rng.randint(8_000_000, 60_000_000)) code = "WEA" if rng.random() < 0.8 else rng.choice(["CSA", "CSA", "AFR"]) if code != "WEA": end = min(end, pos + rng.randint(3_000_000, 12_000_000)) rows.append([f"chr{i + 1}", pos, end, hap, code]) pos = end _, shares = afr.painting_shares(rows) table += [["painting", code, afr.label("continent", code), f"{sh:.4f}", ".", ".", "."] for code, sh in sorted(shares.items(), key=lambda kv: -kv[1])] out = os.path.join(d, "ancestry_fine") os.makedirs(out, exist_ok=True) afr.write_rows(os.path.join(out, f"{SAMPLE}_ancestry_fine.tsv"), ["level", "code", "group", "fraction", "se", "low", "high"], table) with open(os.path.join(out, f"{SAMPLE}_painting.png"), "wb") as f: f.write(afr.painting_png(rows)) def paralogs(d): """Step 35b: a Paraphase JSON with the usual copy numbers (two of each gene, four in each gene family), as bin/paraphase_report.py tables it. Paraphase reads PacBio HiFi; DEMO-001 carries the section so its card is in the pictures.""" import paraphase_report as pr def region(total, rsi, **kw): return dict({"total_cn": total, "genome_depth": MEAN_DEPTH, "region_depth": {"median": (total or 4) * 15.6}, "region_specific_info": rsi}, **kw) data = { "smn1": region(4, {"smn1_cn": 2, "smn2_cn": 2, "smn_del78_cn": 0}), "hba": region(4, {"genotype": "aa/aa", "sv_called": {}}, final_haplotypes={"1": "hba_hba2hap1", "2": "hba_hba2hap2", "3": "hba_hba1hap1", "4": "hba_hba1hap2"}), "rccx": region(4, {"annotated_alleles": ["WT", "WT"], "phasing_success": True}), "GBA": region(4, {}, genes_in_region=["GBA1", "GBAP1"], final_haplotypes={"1": "GBA_gene1hap1", "2": "GBA_gene1hap2", "3": "GBA_gene2hap1", "4": "GBA_gene2hap2"}), "pms2": region(4, {"gene_cn": 2}), "strc": region(4, {"gene_cn": 2}), "ncf1": region(6, {"gene_cn": 2}), "OTOA": region(4, {}, genes_in_region=["OTOA", "OTOAP1"], final_haplotypes={"1": "OTOA_gene1hap1", "2": "OTOA_gene1hap2", "3": "OTOA_gene2hap1", "4": "OTOA_gene2hap2"}), "CFC1": region(4, {}, genes_in_region=["CFC1", "CFC1B"]), "NEB": region(6, {}, genes_in_region=["NEB"]), "TNXB": region(None, {}, genes_in_region=["TNXB"]), } out = os.path.join(d, "paralogs") os.makedirs(out, exist_ok=True) pr.write_tsv(os.path.join(out, f"{SAMPLE}_paraphase_copy_number.tsv"), pr.CN_HEADER, pr.cn_table(data)) pr.write_tsv(os.path.join(out, f"{SAMPLE}_paraphase_variants.tsv"), pr.VARIANT_HEADER, []) # Step 35 (Parascopy, short reads): two SMN1 and two SMN2 copies at exon 7, # in the columns of its _smn_copy_number.tsv. put(os.path.join(out, f"{SAMPLE}_smn_copy_number.tsv"), "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\nchr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t2,2\t33,30\t" "chr5:70049000-70077000:-\n") # Step 35's HBA1/HBA2 table: bin/hba_copy_number.py's own assess() on # made-up depths, four alpha-globin genes. import hba_copy_number as hba depth = {"flank_left": 31.2, "HBA2": 30.9, "HBA1": 30.1, "flank_right": 30.4} res = hba.assess(depth, {k: v * (0.76 if k.startswith("HBA") else 0.98) for k, v in depth.items()}) put(os.path.join(out, f"{SAMPLE}_hba_copy_number.tsv"), "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in hba.ORDER)) def couple(d, rng): """Step 39 with an invented partner, DEMO-002: each carries one change in a different recessive gene of the carrier list, so no gene is shared and the card names none. Rows by bin/couple_report.py's own screen_couple. Only DEMO-001 carries the table: the family page reads one copy of it.""" if SAMPLE != "DEMO-001": return import couple_report genes = couple_report.read_genes(os.path.join(REPO, "assets", "carrier_genes.tsv")) ar = sorted(g for g, v in genes.items() if v["inheritance"] == "AR" and "ACMG" in v["lists"]) found = {} for who, gene in zip((SAMPLE, "DEMO-002"), rng.sample(ar, 2)): g = genes[gene] pos = rng.randint(int(g["start"]), int(g["end"])) ref, alt = rng.sample("ACGT", 2) text = f"{g['chrom']}:{pos} {ref}>{alt} het, Pathogenic, 2 stars" found[who] = [couple_report.finding(gene, text, 1)] rows = couple_report.screen_couple(SAMPLE, "DEMO-002", {SAMPLE: SEX, "DEMO-002": "female"}, found, genes) os.makedirs(os.path.join(d, "couple"), exist_ok=True) couple_report.write_table(os.path.join(d, "couple", f"{SAMPLE}_couple_carrier.tsv"), rows) def consumer_export(d): """Step 40's summary table, in bin/export_consumer.py's rows: the pinned site table keeps 614,138 of the chip list's 638,463 sites (docs/40-export-consumer.md); the counts below are made up.""" sites, from_blocks, from_records = 614138, 470912, 140206 reasons = {"no gVCF record": 1954, "inside a deletion or a longer allele": 611, "filtered call": 309, "no-call in the gVCF": 146} called = from_blocks + from_records assert sites - called == sum(reasons.values()) rows = [("sites", sites), ("called", called), ("no_call", sites - called), ("from_reference_blocks", from_blocks), ("from_variant_records", from_records), ("reference_checked", from_records), ("reference_mismatch", 0), ("template_sites", 638463), ("table_dropped", 24325), ("sex", SEX), ("file_23andme", f"{SAMPLE}_23andme_v5.zip"), ("file_ancestrydna", f"{SAMPLE}_ancestrydna_layout.zip")] rows += sorted((f"no_call: {k}", v) for k, v in reasons.items()) put(os.path.join(d, "consumer_export", f"{SAMPLE}_consumer_export.tsv"), "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows)) # Step 41's rows (bin/bloodtype_report.py's columns and spelling), a common # European set. BLOODTYPE = [ ("ABO", "A1", "A1/O.01", "high", ""), ("RhD", "D+", "RHD/RHD (RHD copies by read depth 1.96)", "high", "weak and partial D (RHD missense alleles) are not assessed"), ("RhCE", "C+c+ E-e+", "C/c e/e", "high", ""), ("Kell", "K-k+ Kp(a-b+) Js(a-b+)", "k/k Kpb/Kpb Jsb/Jsb", "high", ""), ("Duffy", "Fy(a+b+)", "FY*A/FY*B", "high", ""), ("Kidd", "Jk(a+b+)", "Jka/Jkb", "high", ""), ("MNS", "M+N+ S-s+", "M/N s/s", "high", ""), ("Lutheran", "Lu(a-b+)", "Lub/Lub", "high", ""), ("Diego", "Di(a-b+)", "Dib/Dib", "high", ""), ("Dombrock", "Do(a+b+)", "Doa/Dob", "medium", ""), ("Colton", "Co(a+b-)", "Coa/Coa", "high", ""), ("Vel", "Vel+", "Vel/Vel", "high", ""), ("Langereis", "Lan+", "Lan/Lan", "high", ""), ("Junior", "Jr(a+)", "Jra/Jra", "high", ""), ("H", "H+", "H/H", "high", ""), ("Secretor", "secretor", "Se/se", "high", "null allele: FUT2 c.461G>A x1; weak secretor alleles (such as FUT2 c.418A>T, also written c.385A>T) " "and a deletion of the whole FUT2 gene are not assessed"), ("HPA-1", "HPA-1(a+b-)", "1a/1a", "high", ""), ("HPA-2", "HPA-2(a+b-)", "2a/2a", "high", ""), ("HPA-3", "HPA-3(a+b+)", "3a/3b", "high", ""), ("HPA-4", "HPA-4(a+b-)", "4a/4a", "high", ""), ("HPA-5", "HPA-5(a+b-)", "5a/5a", "high", ""), ("HPA-6", "HPA-6(a+b-)", "6a/6a", "high", ""), *[(f"HPA-{n}", f"HPA-{n}(a+b-)", f"{n}a/{n}a", "high", "") for n in range(7, 15)], ("HPA-15", "HPA-15(a+b+)", "15a/15b", "high", ""), *[(f"HPA-{n}", f"HPA-{n}(a+b-)", f"{n}a/{n}a", "high", "") for n in range(16, 36)], ("HNA-1", "HNA-1(a+b+c-)", "FCGR3B*01/FCGR3B*02", "low", "FCGR3B and FCGR3A are near copies and the FCGR3B copy number varies (none to four or more): " "short reads can misplace these calls, the copies are not counted, and HNA-1null (no FCGR3B) " "is not assessed"), ("HNA-3", "HNA-3(a+b-)", "3a/3a", "high", ""), ("HNA-4", "HNA-4(a+b-)", "4a/4a", "high", ""), ("HNA-5", "HNA-5(a+b-)", "5a/5a", "high", ""), ] def bloodtype(d): import bloodtype_report names = [s for s, _ in bloodtype_report.SYSTEMS] if [r[0] for r in BLOODTYPE] != names: raise SystemExit(f"BLOODTYPE lists {[r[0] for r in BLOODTYPE]}, step 41 writes {names}") # A platelet or neutrophil antigen the phenotype lacks gets the marker table's # meaning at the head of the note, as bin/bloodtype_report.py's meaning() writes it. sites, _ = bloodtype_report.read_markers(os.path.join(REPO, "assets", "bloodtype_markers.tsv")) means = {} for r in sites.values(): if r.get("meaning"): means.setdefault(r["system"], r["meaning"]) rows = [] for system, phenotype, genotype, conf, note in BLOODTYPE: lacks = re.findall(r"([a-z])-", phenotype.partition("(")[2]) if system in means and lacks: prefix, num = system.split("-") head = f"lacks {', '.join(f'{prefix}-{num}{a}' for a in lacks)}: {means[system]}" note = "; ".join([head] + ([note] if note else [])) rows.append((system, phenotype, genotype, conf, note)) os.makedirs(os.path.join(d, "bloodtype"), exist_ok=True) bloodtype_report.write_tsv(os.path.join(d, "bloodtype", f"{SAMPLE}_bloodtype.tsv"), bloodtype_report.TABLE_HEADER, [dict(zip(bloodtype_report.TABLE_HEADER, r)) for r in rows]) # Step 42: copies of each row's alt allele, a common European set; the # TAS2R38 rows are one taster and one non-taster haplotype, APOE is e3/e3. # AHR and OR2M7 have no genotype, as a gap in the gVCF. TRAIT_ALT_COPIES = { "rs4988235": 1, "rs182549": 1, "rs145946881": 0, "rs41380347": 0, "rs41525747": 0, "rs671": 0, "rs1229984": 2, "rs762551": 2, "rs2472297": 1, "rs713598": 1, "rs1726866": 1, "rs10246939": 1, "rs17822931": 0, "rs10427255": 1, "rs1815739": 1, "rs12913832": 1, "rs1800407": 0, "rs16891982": 2, "rs1426654": 0, "rs12203592": 0, "rs1805007": 0, "rs1805008": 0, "rs1805009": 0, "rs3827760": 0, "rs1801133": 1, "rs601338": 1, "rs2282679": 1, "rs887829": 1, "rs1801272": 0, "rs429358": 0, "rs7412": 0, "rs10455872": 0, "rs3798220": 0, "rs78655421": 0, } def y_str(d, rng): """Step 45: one row per marker of assets/ystr_panel.tsv from bin/ystr_report.py's marker_row, so the multi-copy, absent and too-long markers read not callable as in a real run; every other marker gets a made-up count at confidence high.""" import ystr_report rows = [] for r in ystr_report.read_panel(os.path.join(REPO, "assets", "ystr_panel.tsv")): row = ystr_report.marker_row(r, {}, 150) if row["confidence"] == "no call": row.update(count=str(rng.randint(10, 30)), confidence="high", reads=str(rng.randint(18, 34)), stutter_reads="0", flank_indel_reads="0", q="0.99", note=".") rows.append(row) put(os.path.join(d, "y_str", f"{SAMPLE}_y_str.tsv"), "\t".join(ystr_report.TSV_COLUMNS) + "\n" + "".join("\t".join(row[k] for k in ystr_report.TSV_COLUMNS) + "\n" for row in rows)) def viral(d): """Step 47: bin/viral_report.py's build_rows on made-up coverage: the lab's phiX control, a few Epstein-Barr reads, no HHV-6, at a human depth of 31.""" import csv import viral_report as vr genomes = [("NC_001664.4", "Human herpesvirus 6A, complete genome", 159378, 0, (0.0, 0.0, 0)), ("NC_000898.1", "Human herpesvirus 6B, complete genome", 162114, 0, (0.0, 0.0, 0)), ("NC_007605.1", "Human gammaherpesvirus 4, complete genome", 171823, 7, (0.0041, 0.006, 0)), ("NC_001422.1", "Escherichia phage phiX174, complete genome", 5386, 1412, (1.0, 38.2, 37))] rows = vr.build_rows([(a, t) for a, t, *_ in genomes], {a: {"reads": n, "length": ln} for a, _, ln, n, _ in genomes}, {a: dep for a, *_, dep in genomes}, 31.0) os.makedirs(os.path.join(d, "viral"), exist_ok=True) with open(os.path.join(d, "viral", f"{SAMPLE}_viral.tsv"), "w", encoding="utf-8", newline="") as f: w = csv.DictWriter(f, fieldnames=vr.HEADER, delimiter="\t", lineterminator="\n") w.writeheader() w.writerows(rows) def lpa(d, rng): """Step 46: bin/lpa_kiv2.py's assess on made-up depths, about 37 KIV-2 repeats at a flank depth near 31, half the KIV-2 depth at MAPQ 0.""" import lpa_kiv2 m, q1 = {}, {} for *_, name in lpa_kiv2.REGIONS: m[name] = rng.uniform(83, 90) if name.startswith("kiv2_") else rng.uniform(29.5, 32.5) q1[name] = m[name] * (0.5 if name.startswith("kiv2_") else 0.98) res = lpa_kiv2.assess(m, q1) res["advice"] = lpa_kiv2.ADVICE put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), "metric\tvalue\n" + "".join(f"{k}\t{res[k]}\n" for k in lpa_kiv2.ORDER)) def chip(d, rng): """Step 49: no CHIP call, and one depth row per gene of assets/chip_genes.tsv at a made-up mean depth near 31, with bin/chip_report.py's own detection limit.""" import chip_report table = chip_report.read_table(os.path.join(REPO, "assets", "chip_genes.tsv")) rows = [] for r in table: mean = rng.uniform(29.0, 33.0) rows.append({"gene": r["gene"], "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_report.detection_limit(mean):.3f}", "status": "covered"}) calls = [] if has("chip"): # One small DNMT3A clone, above the detection limit at this depth (bin/chip_report.py's columns). calls.append({"gene": "DNMT3A", "chrom": "chr2", "pos": "25234373", "ref": "C", "alt": "T", "consequence": "missense_variant", "protein_change": "p.R882H", "exon": "23", "vaf": "0.161", "alt_reads": "8", "depth": "50", "filter": "PASS", "status": "chip", "class": "hotspot", "note": "a published CHIP hotspot"}) put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(chip_report.OUT_COLUMNS) + "\n" + "".join("\t".join(c[k] for k in chip_report.OUT_COLUMNS) + "\n" for c in calls)) put(os.path.join(d, "chip", f"{SAMPLE}_chip_depth.tsv"), "\t".join(chip_report.DEPTH_COLUMNS) + "\n" + "".join("\t".join(x[c] for c in chip_report.DEPTH_COLUMNS) + "\n" for x in rows)) def archaic(d, rng): """Step 43: bin/archaic_report.py's key/value summary, with made-up shares near the published non-African range and five made-up longest segments.""" analysed = 2_870_000_000 nea_n, den_n = rng.randint(420, 520), rng.randint(2, 6) nea_bp, den_bp = int(analysed * rng.uniform(0.017, 0.021)), int(analysed * rng.uniform(0.0002, 0.0006)) rows = [("sample", SAMPLE), ("method", "IBDmix"), ("archaic_genomes", "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan"), ("population", "EUR"), ("min_lod", "4"), ("min_length_bp", "50000"), ("analysed_bp", analysed), ("analysed_chromosomes", 22), ("analysed_sites", rng.randint(5_500_000, 6_500_000)), ("genes", "GENCODE protein-coding"), ("neanderthal_segments", nea_n), ("neanderthal_bp", nea_bp), ("neanderthal_percent", f"{100 * nea_bp / analysed:.2f}"), ("denisovan_segments", den_n), ("denisovan_bp", den_bp), ("denisovan_percent", f"{100 * den_bp / analysed:.2f}")] segs = [("chr1", "Neanderthal"), ("chr3", "Neanderthal"), ("chr6", "Neanderthal"), ("chr9", "Neanderthal"), ("chr12", "Denisovan")] longest = [] for i, (c, anc) in enumerate(segs, 1): s = rng.randint(10_000_000, 90_000_000) n = int(rng.uniform(0.9, 1.3 - 0.08 * i) * 1e6) who = "Altai Denisovan" if anc == "Denisovan" else "Altai/Vindija" longest.append((f"longest_{i}", f"{c}:{s}-{s + n} {anc} {n / 1e6:.2f} Mb ({who}); genes: none")) put(os.path.join(d, "archaic", f"{SAMPLE}_archaic.tsv"), "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in rows + longest)) def ancient(d, rng): """Step 44: bin/ancient_report.py's nearest-group table, info table and PC plot (its own svg_plot), made-up distances and PCs over a few AADR-style group labels.""" import ancient_report groups = [("England_BellBeaker", "2450-2000 BCE", "United Kingdom", "Amesbury Down"), ("Germany_CordedWare", "2900-2350 BCE", "Germany", "Esperstedt"), ("Spain_BellBeaker", "2500-2000 BCE", "Spain", "Camino de las Yeseras"), ("Czechia_EBA_Unetice", "2200-1700 BCE", "Czech Republic", "Moravia"), ("Hungary_LBA", "1500-1100 BCE", "Hungary", "Tiszafüred"), ("Italy_Imperial_Rome", "1-300 CE", "Italy", "Rome"), ("Sweden_Viking", "800-1050 CE", "Sweden", "Sigtuna"), ("France_Neolithic", "4500-3000 BCE", "France", "Gurgy"), ("Turkey_Neolithic", "6500-6000 BCE", "Turkey", "Barcın"), ("Russia_Yamnaya_Samara", "3300-2700 BCE", "Russia", "Samara")] person = (0.01234, -0.00456) dist, nearest = 0.0, [] lines = ["rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"] for i, (g, dates, country, loc) in enumerate(groups, 1): dist += rng.uniform(0.002, 0.012) pc1, pc2 = person[0] + rng.uniform(-dist, dist), person[1] + rng.uniform(-dist, dist) nearest.append({"group": g, "pc1": pc1, "pc2": pc2}) lines.append(f"{i}\t{g}\t{rng.randint(3, 40)}\t{dist:.5f}\t{pc1:.5f}\t" f"{pc2:.5f}\t{rng.randint(1000, 8500)}\t{dates}\t{country}\t{loc}\n") put(os.path.join(d, "ancient", f"{SAMPLE}_ancient.tsv"), "".join(lines)) panel = {f"m{i}": ("present", (rng.gauss(0.0, 0.03), rng.gauss(0.0, 0.025))) for i in range(300)} ancients = {} for i in range(200): panel[f"a{i}"] = (ancient_report.ANCIENT_LABEL, (rng.gauss(0.005, 0.035), rng.gauss(0.0, 0.03))) ancients[f"a{i}"] = {"date_bp": str(rng.randint(500, 9000))} put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_pca.svg"), ancient_report.svg_plot(person, panel, ancients, SAMPLE, nearest)) info = [("release", "v62.0"), ("sites_used", rng.randint(520_000, 560_000)), ("panel_sites", 597_573), ("present_day_people", 819), ("ancient_people", 4_210), ("ancient_groups", 612), ("pcs_in_distance", 4), ("pc1", f"{person[0]:.5f}"), ("pc2", f"{person[1]:.5f}")] put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_info.tsv"), "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in info)) def exomiser(d): """Step 48: bin/exomiser_report.py's top_rows and words on made-up Exomiser rows for two searched symptoms: two weak candidates of uncertain significance, the shape of a search that finds nothing to act on.""" import exomiser_report as er terms = [{"hpo_id": "HP:0001250", "label": "Seizure", "given_as": "HP:0001250"}, {"hpo_id": "HP:0002315", "label": "Headache", "given_as": "HP:0002315"}] def row(rank, gene, moi, scores, contig, start, ref, alt, gt, cls, hgvs, disease_id, disease): r = dict.fromkeys(er.NEED, "") r.update({"#RANK": str(rank), "GENE_SYMBOL": gene, "MOI": moi, "P-VALUE": scores[3], "EXOMISER_GENE_COMBINED_SCORE": scores[0], "EXOMISER_GENE_PHENO_SCORE": scores[1], "EXOMISER_GENE_VARIANT_SCORE": scores[2], "CONTRIBUTING_VARIANT": "1", "CONTIG": contig, "START": str(start), "REF": ref, "ALT": alt, "GENOTYPE": gt, "FUNCTIONAL_CLASS": cls, "HGVS": hgvs, "EXOMISER_ACMG_CLASSIFICATION": "UNCERTAIN_SIGNIFICANCE", "EXOMISER_ACMG_DISEASE_ID": disease_id, "EXOMISER_ACMG_DISEASE_NAME": disease, "CLINVAR_PRIMARY_INTERPRETATION": "NOT_PROVIDED", "MAX_FREQ_SOURCE": "GNOMAD_E_NFE", "MAX_FREQ": "0.0123"}) return r rows = er.top_rows([ row(1, "SCN1A", "AD", ("0.412", "0.788", "0.534", "0.021"), "2", 165991230, "C", "T", "0/1", "missense_variant", "SCN1A:ENST00000674923.1:c.3199G>A:p.(Ala1067Thr)", "OMIM:607208", "Dravet syndrome"), row(2, "CACNA1A", "AD", ("0.297", "0.701", "0.421", "0.048"), "19", 13299371, "G", "A", "0/1", "missense_variant", "CACNA1A:ENST00000360228.11:c.2134C>T:p.(Arg712Cys)", "OMIM:141500", "Migraine, familial hemiplegic, 1")], 10) out = os.path.join(d, "exomiser") put(os.path.join(out, f"{SAMPLE}_exomiser_terms.tsv"), "\t".join(er.TERMS_HEADER) + "\n" + "".join("\t".join(t[k] for k in er.TERMS_HEADER) + "\n" for t in terms)) put(os.path.join(out, f"{SAMPLE}_exomiser_top.tsv"), "\t".join(er.OUT_HEADER) + "\n" + "".join("\t".join(r[k] for k in er.OUT_HEADER) + "\n" for r in rows)) put(os.path.join(out, f"{SAMPLE}_exomiser.txt"), "\n".join(er.words(rows, terms)) + "\n") def traits(d): """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by bin/traits_report.py.""" import traits_report rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) unknown = sorted(r["rsid"] for r in rows if r["rsid"] not in TRAIT_ALT_COPIES | {"rs4410790": 0, "rs4481887": 0}) if unknown: raise SystemExit(f"assets/traits.tsv has rows TRAIT_ALT_COPIES does not give: {', '.join(unknown)}") recs = {} for r in rows: if r["rsid"] in TRAIT_ALT_COPIES: n = TRAIT_ALT_COPIES[r["rsid"]] recs[(r["chrom"], r["pos"])] = [(r["ref"], [r["ref"]] * (2 - n) + [r["alt"]] * n, "60")] os.makedirs(os.path.join(d, "traits"), exist_ok=True) traits_report.write_tsv(os.path.join(d, "traits", f"{SAMPLE}_traits.tsv"), traits_report.report(rows, recs)) def mocha(d): """Step 51: bin/mocha_report.py's report on made-up MoChA tables for a 30x genome with no mosaic call: chr1 to chr22 read, the sample filter met.""" import mocha_report as mr stats = [{"sample_id": SAMPLE, "call_rate": "0.9981", "cov_median": "31", "baf_auto": "0.0124", "n_hets": "2100000"}] sites = [(f"chr{i}", 10001, 10001 + 100_000_000, 120_000, 95_000, 94_000) for i in range(1, 23)] calls = [] if has("mocha"): # A loss of 20q in about 12% of the blood cells, which MoChA's filters keep and the floor lets through. calls.append({"sample_id": SAMPLE, "chrom": "chr20", "beg_GRCh38": "30100000", "end_GRCh38": "42300000", "length": "12200000", "p_arm": "N", "q_arm": "Y", "type": "Loss", "cf": "0.120", "bdev": "0.031", "bdev_se": "0.004", "n50_hets": "350000", "lod_baf_conc": "nan", "rel_cov": "1.86", "lod_baf_phase": "48.2", "lod_lrr_baf": "35.1", "n_hets": "4210"}) rows, summary = mr.report(SAMPLE, calls, stats, sites, "beagle") os.makedirs(os.path.join(d, "mosaic"), exist_ok=True) with open(os.path.join(d, "mosaic", f"{SAMPLE}_mocha.tsv"), "w", encoding="utf-8") as f: f.write("\t".join(mr.CALL_COLUMNS) + "\n") for x in rows: f.write("\t".join(str(x[k]) for k in mr.CALL_COLUMNS) + "\n") with open(os.path.join(d, "mosaic", f"{SAMPLE}_mocha_summary.tsv"), "w", encoding="utf-8") as f: f.write("metric\tvalue\n") for k, v in summary.items(): f.write(f"{k}\t{v}\n") def vntyper(d): """Step 52: bin/vntyper_report.py's reading of a made-up VNtyper run with no MUC1 frameshift at a depth its coverage check passes.""" import tempfile import vntyper_report as vr with tempfile.TemporaryDirectory() as run: put(os.path.join(run, "kestrel", "kestrel_result.tsv"), "Motif\tVariant\tConfidence\tFlag\nnone\tNone\tNegative\tNot flagged\n") put(os.path.join(run, "coverage", "coverage_summary.tsv"), "mean\tpercent_uncovered\tcoverage_qc\n212.4\t0.0\tPASS\n") t = vr.read(run, 0) os.makedirs(os.path.join(d, "vntyper"), exist_ok=True) with open(os.path.join(d, "vntyper", f"{SAMPLE}_vntyper.tsv"), "w", encoding="utf-8") as f: f.write("metric\tvalue\n") for k in vr.FIELDS: f.write(f"{k}\t{t[k]}\n") def locityper(d, rng): """Step 53: bin/locityper_report.py's reading of a made-up Locityper run that called every locus of assets/locityper_loci.bed with two HPRC panel haplotypes and passed the authors' filter.""" import gzip import tempfile import locityper_report as lr loci_path = os.path.join(REPO, "assets", "locityper_loci.bed") panel = ["HG00438.1", "HG00733.2", "HG01891.1", "HG02257.2", "NA18906.1", "NA19240.2"] out = os.path.join(d, "locityper") os.makedirs(out, exist_ok=True) with tempfile.TemporaryDirectory() as run: for name, _ in lr.read_loci(loci_path): res = {"genotype": ",".join(rng.sample(panel, 2)), "quality": round(rng.uniform(25, 60), 1), "total_reads": rng.randint(800, 4000), "unexpl_reads": rng.randint(0, 30), "weight_dist": round(rng.uniform(0.1, 3), 3)} put(os.path.join(run, "loci", name, "success"), "") with gzip.open(os.path.join(run, "loci", name, "res.json.gz"), "wt") as f: json.dump(res, f) if lr.main(["--gt-dir", run, "--loci", loci_path, "--out", os.path.join(out, f"{SAMPLE}_locityper.tsv")]): raise SystemExit("locityper_report failed") def mei(d, rng): """Step 50: bin/mei_report.py run on a made-up MEGAnE VCF (a few Alu and L1 insertions away from the catalog) and a made-up bedcov near 31x over every window of assets/mei_sites.tsv, so every catalog row reads "not seen".""" import tempfile import mei_report sites_path = os.path.join(REPO, "assets", "mei_sites.tsv") sites = mei_report.read_sites(sites_path) out = os.path.join(d, "mei") os.makedirs(out, exist_ok=True) with tempfile.TemporaryDirectory() as t: vcf, depth = os.path.join(t, "megane.vcf"), os.path.join(t, "depth.tsv") recs = [("chr3", 41258120, "SINE/Alu", "AluYa5"), ("chr7", 102931455, "SINE/Alu", "AluYb8"), ("chr11", 63044871, "LINE/L1", "L1HS")] put(vcf, "##fileformat=VCFv4.1\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + SAMPLE + "\n" + "".join(f"{c}\t{p}\t.\tN\t\t.\tPASS\tSVTYPE={e};MEPRED=PASS;MEI={s};CN_conf=H\tGT:CN\t0/1:1\n" for c, p, e, s in recs)) put(depth, "".join(f"{r['chrom']}\t{r['start']}\t{r['end']}\t{r['gene']}\t" f"{round((int(r['end']) - int(r['start'])) * rng.uniform(28, 34))}\t500\n" for r in sites)) if mei_report.main(["--sample", SAMPLE, "--vcf", vcf, "--sites", sites_path, "--depth", depth, "--out-calls", os.path.join(out, f"{SAMPLE}_mei.tsv"), "--out-sites", os.path.join(out, f"{SAMPLE}_mei_sites.tsv")]): raise SystemExit("mei_report failed") def run_status(d, started): """logs/run_status.tsv as run-all.sh writes it, every step ok.""" lines = ["# run-all.sh: when this run started and how each step ended", f"meta\tstarted_epoch\t{int(started)}", f"meta\tstarted_utc\t{time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(started))}", f"meta\tdeclared_sex\t{SEX}"] lines += [f"step\t{s}\tok" for s in sorted(collect_summary.RUN_ALL_STEPS)] put(os.path.join(d, "logs", "run_status.tsv"), "\n".join(lines) + "\n") # --- the two commands ---------------------------------------------------------------- def make(a): global SAMPLE, SEX SAMPLE, SEX = a.sample, a.sex if not re.match(r"^DEMO-[0-9]{3}$", SAMPLE): raise SystemExit(f"--sample {SAMPLE}: an invented name of the form DEMO-001") # A fixed start, so the report's "Latest run" line is the same on every # retake; the files written now are newer, so none is marked stale. started = RUN_STARTED d = os.path.join(a.genome_dir, SAMPLE) if os.path.exists(d): raise SystemExit(f"{d} exists: give an empty --genome-dir") os.makedirs(d) run_status(d, started) # Each part gets its own generator, so changing one leaves the others' values alone. parts = [("variants", lambda r: variants(d, r, a.records)), ("clinvar", lambda r: clinvar(d, a.genome_dir)), ("pgx", lambda r: pgx(d, a.genome_dir, r)), ("prs", lambda r: prs(d, a.genome_dir, r)), ("coverage", lambda r: coverage(d, r)), ("sample_qc", lambda r: sample_qc(d)), ("reads", lambda r: reads(d, r)), ("structural", lambda r: structural(d, r)), ("repeats", lambda r: repeats_and_more(d, r)), ("clinical", lambda r: clinical(d, r)), # after "reads" and "sample_qc", whose files these two read ("read_count", lambda r: read_count(d)), ("mosaic", lambda r: mosaic(d, r)), ("ancestry_fine", lambda r: ancestry_fine(d, r)), ("paralogs", lambda r: paralogs(d)), ("couple", lambda r: couple(d, r)), ("consumer_export", lambda r: consumer_export(d)), ("bloodtype", lambda r: bloodtype(d)), ("traits", lambda r: traits(d)), ("y_str", lambda r: y_str(d, r) if SEX == "male" else None), ("viral", lambda r: viral(d)), ("lpa", lambda r: lpa(d, r)), ("chip", lambda r: chip(d, r)), ("archaic", lambda r: archaic(d, r)), ("ancient", lambda r: ancient(d, r)), ("exomiser", lambda r: exomiser(d)), ("mocha", lambda r: mocha(d)), ("vntyper", lambda r: vntyper(d)), ("mei", lambda r: mei(d, r)), ("locityper", lambda r: locityper(d, r)), ("callability", lambda r: callability(d, r))] for name, fn in parts: fn(random.Random(f"{a.seed}:{name}")) print(f" {name}: written") print(f"{SAMPLE} written in {d}") return 0 def check(a): """Fail when a section of step 24's summary is not ok.""" path = os.path.join(a.genome_dir, SAMPLE, "summary.json") with open(path) as f: s = json.load(f) bad = [(k, v["state"], v.get("note") or "") for k, v in s["sections"].items() if v["state"] != "ok"] for k, v in s["sections"].items(): print(f" {k:<14} {v['state']}") if s["cyp2d6"].get("agree") is not True: bad.append(("cyp2d6", "callers do not agree", json.dumps(s["cyp2d6"]["calls"]))) if bad: for k, st, note in bad: print(f"FAIL: {k} is {st} {note}".rstrip(), file=sys.stderr) print("A card would say 'Not run' or worse: feed that section in tests/demo/make_demo_sample.py.", file=sys.stderr) return 1 print(f"Every section of {path} is ok.") return 0 def main(argv=None): argv = sys.argv[1:] if argv is None else argv if argv[:1] == ["check"]: ap = argparse.ArgumentParser(prog="make_demo_sample.py check") ap.add_argument("--genome-dir", required=True) return check(ap.parse_args(argv[1:])) ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--genome-dir", required=True, help="an empty folder; DEMO-001 is written inside it") ap.add_argument("--seed", type=int, default=1) ap.add_argument("--records", type=int, default=4_700_000, help="variant records in the VCF (default 4700000)") ap.add_argument("--sample", default="DEMO-001", help="the invented sample to write (default DEMO-001)") ap.add_argument("--sex", default="male", choices=("male", "female")) return make(ap.parse_args(argv)) if __name__ == "__main__": sys.exit(main())