---
work_path: ../work
log_level: DEBUG
pool_size: 4

scratchpad:
  string_version: '12.5'

steps:
  #: VARIANT PPP STEP :############################################################################
  variant_ppp:
    - name: transform LoF curation annotations
      source: input/variant/lof_curation/curation.csv
      destination: intermediate/variant/lof_curation
      transformer: lof_curation
  ##################################################################################################

  #: VARIANT STEP :################################################################################
  variant:
    - name: transform FoldX annotations
      source: input/variant/foldx
      destination: intermediate/variant/foldx
      transformer: foldx
      settings:
        plddt_threshold: 0.7

    - name: transform variant index
      requires:
        - transform FoldX annotations
      transformer: variant
      source:
        uniprot_variants: input/evidence/uniprot_variants.json.gz
        eva: input/evidence/eva.json.gz
        pharmacogenomics: output/pharmacogenomics
        credible_set_variant: intermediate/validated/credible_set_variant
        fasta: input/variant/reference/Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz
        gff3: input/variant/reference/Homo_sapiens.GRCh38.116.gff3.gz
        gerp: input/variant/gerp/gerp_conservation_scores.homo_sapiens.GRCh38.bw
        vep_cache: input/variant/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz
        sift_predictions: input/variant/variation/protein_function_predictions.txt.gz
        sift_translation_md5: input/variant/variation/translation_md5.txt.gz
        sift_attrib: input/variant/variation/attrib.txt.gz
        peptides: input/variant/reference/Homo_sapiens.GRCh38.pep.all.fa.gz
        alphamissense: input/variant/alphamissense/AlphaMissense_hg38.tsv.gz
        alphamissense_isoforms: input/variant/alphamissense/AlphaMissense_isoforms_hg38.tsv.gz
        ancestor: input/variant/loftee/human_ancestor.fa.gz
        core: input/ensembl/homo_sapiens
        variant_annotations:
          - input/variant/gnomad
        amino_acid_annotations:
          - intermediate/variant/foldx
        gpn_star: input/variant/gpn_star
      destination:
        output: output/variant
      settings:
        hash_threshold: 300
        parallel_chromosomes: 8
        parallel_chunks: 32
        # songlab/gpn-star-scores@204d08a92bdf07117f5d869db667c45f9029ed48: genome-wide quantiles 0.1 / 0.01 / 0.001 and minimum
        # measured with cutoffs_from_histogram(llr_histogram(sorted(gpn_star_files(<mirror>).values())))
        gpn_star:
          top_10_percent: -2.392
          top_1_percent: -5.448
          top_0_1_percent: -9.851
          minimum: -13.698
  ##################################################################################################

  #: DISEASE STEP :#################################################################################
  disease:
    - name: transform disease_efo
      source: input/disease/efo_otar_slim.json
      destination: output/disease
      transformer: disease

    - name: transform disease_hpo
      source: input/disease/hp-full.json
      destination: output/disease_hpo
      transformer: disease_hpo

    - name: transform disease_phenotype
      requires:
        - transform disease_efo
      source:
        disease_path: output/disease
        phenotype_path: input/disease/phenotype.hpoa
        mondo_path: input/disease/mondo.json
      destination: output/disease_phenotype
      transformer: disease_phenotype
  ##################################################################################################

  #: STUDY MIGRATION
  study:
    - name: transform validate study
      source:
        study:
          - input/gwas/gcca/study
          - input/gwas/gcss_pics/study
          - input/gwas/gcss_susie/study
          - input/gwas/finngen_gwas/study
          - input/molqtl/eqtl_catalogue/study
          - input/molqtl/ukb_ppp_eur/study
          - input/molqtl/finngen_multiome/study
        target: output/target
        disease: output/disease
        biosample: output/biosample
        heritability: input/heritability_estimate
      destination:
        output: output/study
        excluded: excluded/study
      transformer: study
      settings:
        invalid_qc_reasons:
          - UNRESOLVED_TARGET
          - UNRESOLVED_DISEASE
          - UNKNOWN_STUDY_TYPE
          - DUPLICATED_STUDY
          - UNKNOWN_BIOSAMPLE
          - FAILED_MEAN_BETA_CHECK
          - FAILED_PZ_CHECK
          - FAILED_GC_LAMBDA_CHECK
          - NO_OT_CURATION
          - CASE_CASE_STUDY_DESIGN
          - DEPRECATED_PROJECT
        deprecated_project_ids:
          - GTEx

  #: GO STEP :######################################################################################
  go:
    - name: transform go
      transformer: go
      source: input/go/go.obo
      destination: output/go
  ##################################################################################################

  #: CREDIBLE_SET STEP :###########################################################################
  # Set QC and variant QC run per source: the explode iterates over the source folders below
  # input/ (<kind>/<source>), which the QC output mirrors under intermediate/qc/. Validation runs
  # the cross-set QC over every source, then validates each source's sets against its own variants.
  # The per-source paths are listed explicitly, in the order of the explode.
  credible_set:
    - name: explode qc credible sets
      foreach:
        - gwas/gcca
        - gwas/gcss_pics
        - gwas/gcss_susie
        - gwas/finngen_gwas
        - molqtl/eqtl_catalogue
        - molqtl/ukb_ppp_eur
        - molqtl/finngen_multiome
      do:
        - name: transform qc credible sets ${each}
          transformer: credible_set.set_qc
          source:
            credible_set: input/${each}/credible_set
            study: output/study
          destination: intermediate/qc/${each}/credible_set
          settings:
            allowed_fm_methods:
              - PICS
              - SuSiE
              - SuSiE-inf

        - name: transform qc credible set variants ${each}
          transformer: credible_set.variant_qc
          source:
            credible_set_variant: input/${each}/credible_set_variant
          destination: intermediate/qc/${each}/credible_set_variant
          settings:
            coverage: 0.95
            pip_tolerance: 0.0001

    - name: transform validate credible sets
      requires:
        - explode qc credible sets
      transformer: credible_set.validation
      source:
        credible_set:
          - intermediate/qc/gwas/gcca/credible_set
          - intermediate/qc/gwas/gcss_pics/credible_set
          - intermediate/qc/gwas/gcss_susie/credible_set
          - intermediate/qc/gwas/finngen_gwas/credible_set
          - intermediate/qc/molqtl/eqtl_catalogue/credible_set
          - intermediate/qc/molqtl/ukb_ppp_eur/credible_set
          - intermediate/qc/molqtl/finngen_multiome/credible_set
        credible_set_variant:
          - intermediate/qc/gwas/gcca/credible_set_variant
          - intermediate/qc/gwas/gcss_pics/credible_set_variant
          - intermediate/qc/gwas/gcss_susie/credible_set_variant
          - intermediate/qc/gwas/finngen_gwas/credible_set_variant
          - intermediate/qc/molqtl/eqtl_catalogue/credible_set_variant
          - intermediate/qc/molqtl/ukb_ppp_eur/credible_set_variant
          - intermediate/qc/molqtl/finngen_multiome/credible_set_variant
      destination:
        valid_set: intermediate/validated/credible_set
        valid_variant: intermediate/validated/credible_set_variant
        excluded_set: excluded/credible_set
        excluded_variant: excluded/credible_set_variant
      settings:
        cs_invalid_qc_reasons:
          - INVALID_CS_ID
          - IN_MHC
          - INVALID_CHROMOSOME
          - LEAD_CHROMOSOME_MISMATCH
          - TOP_HIT_AND_SUMMARY_STATS
          - MISSING_STUDY
          - REDUNDANT_PICS_TOP_HIT
          - DUPLICATED_CREDIBLE_SET_ID_ACROSS_SOURCES
          - INVALID_FINEMAPPING_METHOD
          - EXPLAINED_BY_SUSIE
          - COMPOSITE_FLAG
          - INCONSISTENCY_FLAG
          - SUBSIGNIFICANT_FLAG
          - LD_CLUMPED
          - WINDOW_CLUMPED
          - STUDY_TYPE_MISMATCH
          - LEAD_VARIANT_MISSING
        # ABNORMAL_PIP is not listed: sets with abnormal PIP sums are flagged and kept.
        csv_set_invalid_qc_reasons:
          - INVALID_CS_ID
          - INVALID_VARIANT_ID
          - DUPLICATED_VARIANT
        csv_row_invalid_qc_reasons:
          - EXCEEDS_COVERAGE_95
          - ORPHAN_VARIANT
  ##################################################################################################

  #: CREDIBLE SET ANNOTATION STEP :#################################################################
  # Runs after the variant index: the effect allele frequency comes from its gnomAD frequencies, and
  # the variant index reads the validated variants of the credible_set step. The target and disease
  # indexes, for the trans-QTL and replication flags, are upstream of the study step.
  credible_set_annotation:
    - name: transform annotate credible sets
      transformer: credible_set.annotation
      source:
        credible_set: intermediate/validated/credible_set
        credible_set_variant: intermediate/validated/credible_set_variant
        study: output/study
        variant: output/variant
        target: output/target
        disease: output/disease
      destination:
        credible_set: output/credible_set
        credible_set_variant: output/credible_set_variant
      settings:
        trans_qtl_threshold: 5000000
  ##################################################################################################

  #: OTAR STEP :####################################################################################
  otar:
    - name: transform otar
      transformer: otar
      source:
        diseases: output/disease
        otar_meta: input/otar/otar_meta.csv
        otar_project_to_efo: input/otar/otar_project_to_efo.csv
      destination: output/otar
  ##################################################################################################

  #: REACTOME STEP :################################################################################
  reactome:
    - name: transform reactome
      transformer: reactome
      source:
        pathways: input/reactome/ReactomePathways.txt
        relations: input/reactome/ReactomePathwaysRelation.txt
      destination: output/reactome
  ##################################################################################################

  #: BIOSAMPLE STEP :###############################################################################
  biosample:
    - name: transform biosample
      transformer: biosample
      source:
        cell_ontology: input/biosample/cl.json
        uberon: input/biosample/uberon.json
        efo: input/biosample/efo.json
      destination: output/biosample
  ##################################################################################################

  #: EXPRESSION STEP :##############################################################################
  baseline_expression_dice:
    - name: pyspark baseline expression harmonise dice
      pyspark: baseline_expression_harmonise
      source:
        dice_directory: input/baseline_expression/bulkRNAseq/DICE
        mapping_path: input/baseline_expression/bulkRNAseq/DICE/DICE_ontology_mapping.txt
      destination:
        baseline_expression: intermediate/baseline_expression/unaggregated/DICE/celltype
      settings:
        harmoniser: dice
    - name: pyspark baseline expression specificity dice
      pyspark: baseline_expression_specificity
      source: intermediate/baseline_expression/unaggregated/DICE/celltype
      destination: intermediate/baseline_expression/specificity/DICE/celltype
      settings:
        specificity_method: cellex
        biosample: celltype
      requires:
        - pyspark baseline expression harmonise dice
    - name: pyspark baseline expression aggregate dice
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/DICE/celltype/parquet
        celltype_cellex: intermediate/baseline_expression/specificity/DICE/celltype/cellex/esmu.csv.gz
      destination: intermediate/baseline_expression/aggregated/DICE/celltype/
      requires:
        - pyspark baseline expression harmonise dice
        - pyspark baseline expression specificity dice

  baseline_expression_cellxgene:
    - name: pyspark baseline expression harmonise cellxgene
      pyspark: baseline_expression_harmonise
      source:
        h5ad_path: input/baseline_expression/scRNAseq/CxG-tabula_sapiens.h5ad
      destination:
        baseline_expression: intermediate/baseline_expression/unaggregated/tabula_sapiens/
      settings:
        harmoniser: cellxgene
        datasource_id: tabula_sapiens
        datatype_id: scrna-seq
        min_cells: 5 # Minimum cells a gene must be expressed in to be retained
        min_genes: 5 # Minimum genes a cell must express to be retained
        technology: 10X # Technology used to generate the data
        normalise: true
        aggregation_min_cells: 5 # Minimum cells required per pseudobulked sample
        aggregation_method: sum # Options: sum, mean
        tissue_agg_colname: tissue_ontology_term_id
        celltype_agg_colname: cell_type_ontology_term_id
        age_colname: development_stage
        sex_colname: sex
        ethnicity_colname: self_reported_ethnicity
        donor_colname: donor_id
    - name: pyspark baseline expression specificity celltype
      pyspark: baseline_expression_specificity
      source: intermediate/baseline_expression/unaggregated/tabula_sapiens/celltype
      destination: intermediate/baseline_expression/specificity/tabula_sapiens/celltype
      settings:
        specificity_method: cellex
        biosample: celltype
      requires:
        - pyspark baseline expression harmonise cellxgene
    - name: pyspark baseline expression specificity tissue
      pyspark: baseline_expression_specificity
      source: intermediate/baseline_expression/unaggregated/tabula_sapiens/tissue
      destination: intermediate/baseline_expression/specificity/tabula_sapiens/tissue
      settings:
        specificity_method: cellex
        biosample: tissue
      requires:
        - pyspark baseline expression harmonise cellxgene
    - name: pyspark baseline expression specificity tissue celltype
      pyspark: baseline_expression_specificity
      source: intermediate/baseline_expression/unaggregated/tabula_sapiens/tissue_celltype
      destination: intermediate/baseline_expression/specificity/tabula_sapiens/tissue_celltype
      settings:
        specificity_method: cellex
        biosample: tissuecelltype
      requires:
        - pyspark baseline expression harmonise cellxgene
    - name: pyspark baseline expression aggregate celltype
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/tabula_sapiens/celltype/parquet
        celltype_cellex: intermediate/baseline_expression/specificity/tabula_sapiens/celltype/cellex/esmu.csv.gz
      destination: intermediate/baseline_expression/aggregated/tabula_sapiens/celltype
      requires:
        - pyspark baseline expression harmonise cellxgene
        - pyspark baseline expression specificity celltype
    - name: pyspark baseline expression aggregate tissue
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/tabula_sapiens/tissue/parquet
        tissue_cellex: intermediate/baseline_expression/specificity/tabula_sapiens/tissue/cellex/esmu.csv.gz
      destination: intermediate/baseline_expression/aggregated/tabula_sapiens/tissue
      requires:
        - pyspark baseline expression harmonise cellxgene
        - pyspark baseline expression specificity tissue
    - name: pyspark baseline expression aggregate tissue celltype
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/tabula_sapiens/tissue_celltype/parquet
        tissue_celltype_cellex: intermediate/baseline_expression/specificity/tabula_sapiens/tissue_celltype/cellex/esmu.csv.gz
      destination: intermediate/baseline_expression/aggregated/tabula_sapiens/tissue_celltype
      requires:
        - pyspark baseline expression harmonise cellxgene
        - pyspark baseline expression specificity tissue celltype

  baseline_expression_pride:
    - name: pyspark baseline expression harmonise pride
      pyspark: baseline_expression_harmonise
      source:
        pride_directory: input/baseline_expression/MSproteomics/PRIDE
        tissue_ontology_mapping_path: input/baseline_expression/MSproteomics/PRIDE/tissue-complete_mapping.tsv
      destination:
        baseline_expression: intermediate/baseline_expression/unaggregated/PRIDE/tissue
      settings:
        harmoniser: pride
        pride_codes:
          - PXD020192
          - PXD000865
          - PXD000561
          - PXD016999
    - name: pyspark baseline expression aggregate pride
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/PRIDE/tissue/parquet
      destination: intermediate/baseline_expression/aggregated/PRIDE/tissue
      requires:
        - pyspark baseline expression harmonise pride

  baseline_expression_gtex:
    - name: pyspark baseline expression harmonise gtex
      pyspark: baseline_expression_harmonise
      source:
        gtex_source_data_path: input/baseline_expression/bulkRNAseq/GTEx/gene_tpm.gct.gz
        sample_metadata_path: input/baseline_expression/bulkRNAseq/GTEx/sample_attributes.txt
        subject_metadata_path: input/baseline_expression/bulkRNAseq/GTEx/subject_phenotypes.txt
      destination:
        baseline_expression: intermediate/baseline_expression/unaggregated/GTEx/tissue
      settings:
        harmoniser: gtex
        matrix: true # Generate a matrix as well as a parquet (true or false)
    - name: pyspark baseline expression specificity gtex
      pyspark: baseline_expression_specificity
      source: intermediate/baseline_expression/unaggregated/GTEx/tissue
      destination: intermediate/baseline_expression/specificity/GTEx/tissue
      settings:
        specificity_method: cellex
        biosample: tissue
        input_format: true
        mode: matrix
        sample_id: OrigSample
      requires:
        - pyspark baseline expression harmonise gtex
    - name: pyspark baseline expression aggregate gtex
      pyspark: baseline_expression_aggregate
      source:
        expression_data: intermediate/baseline_expression/unaggregated/GTEx/tissue/parquet
        tissue_cellex: intermediate/baseline_expression/specificity/GTEx/tissue/cellex/esmu.csv.gz
      destination: intermediate/baseline_expression/aggregated/GTEx/tissue
      requires:
        - pyspark baseline expression harmonise gtex
        - pyspark baseline expression specificity gtex

  baseline_expression:
    - name: pyspark baseline expression aggregated
      pyspark: baseline_expression
      source:
        baseline_expression: intermediate/baseline_expression
        target_index: output/target
        biosample_index: output/biosample
        tissue_parents: input/baseline_expression/tissue-parent_mapping.tsv
        celltype_parents: input/baseline_expression/celltype-parent_mapping.tsv
      destination:
        valid: output/baseline_expression
        failed: excluded/baseline_expression
      settings:
        partition_count: 39
        aggregation: aggregated
        datasets:
          - GTEx/tissue
          - PRIDE/tissue
          - tabula_sapiens/celltype
          - tabula_sapiens/tissue
          - tabula_sapiens/tissue_celltype
          - DICE/celltype
  ##################################################################################################

  #: MOUSE_PHENOTYPES STEP :########################################################################
  mouse_phenotype:
    - name: transform IMPC into mouse phenotypes split by target
      transformer: mouse_phenotype
      source:
        target: output/target
        # IMPC source files
        solr_gene_gene: input/impc/gene_gene.csv
        solr_mouse_model: input/impc/mouse_model.csv
        solr_disease_model_summary: input/impc/disease_model_summary.csv
        solr_ontology: input/impc/ontology.csv
        hgnc_gene_mappings: input/target/genenames/hgnc_complete_set.tsv
        mouse_gene_mappings: input/impc/MGI_Gene_Model_Coord.rpt
        mouse_pubmed_refs: input/impc/MGI_PhenoGenoMP.rpt
        mp_ontology: input/impc/mp.json
      destination:
        output: output/mouse_phenotype
        excluded: excluded/mouse_phenotype
  ##################################################################################################

  #: CHEMICAL PROBES STEP :#########################################################################
  chemical_probes:
    - name: transform chemical probes
      transformer: chemical_probes
      source:
        probes_excel: input/target/chemicalprobes/probes.xlsx
        drugs_csv: input/target/chemicalprobes/probes_links.csv
        chembl_molecule: intermediate/chembl_molecule
        target: output/target
      destination: output/chemical_probe
  ##################################################################################################

  #: CODING VARIANT STEP :##########################################################################
  coding_variant:
    - name: transform coding_variant view
      transformer: coding_variant
      source:
        variant: output/variant
        target: output/target
        disease: output/disease
        evidence_eva: output/evidence_eva
        evidence_eva_somatic: output/evidence_eva_somatic
        evidence_uniprot_variant: output/evidence_uniprot_variants
        evidence_gwas_credible_set: output/evidence_gwas_credible_sets
        credible_set: output/credible_set
        credible_set_variant: output/credible_set_variant
        pharmacogenetics: output/pharmacogenomics
      destination: view/coding_variant
  ##################################################################################################

  #: PHARMACOVIGILANCE STEP :#######################################################################
  pharmacovigilance:
    - name: transform pharmacovigilance
      transformer: pharmacovigilance
      source:
        drug_molecule: output/drug_molecule
        fda_events: input/openfda
        blacklisted_events: input/openfda/blacklisted_events.txt
      destination: output/pharmacovigilance
      settings:
        montecarlo_permutations: 500
        montecarlo_percentile: 0.95
        archive_workers: 8
  ##################################################################################################

  #: SO STEP :######################################################################################
  so:
    - name: transform so
      source: input/so/so.json
      destination: output/so
      transformer: so
  ##################################################################################################

  #: TARGET STEP :##################################################################################
  target:
    - name: transform target
      transformer: target
      source:
        ensembl: input/ensembl/homo_sapiens
        chembl: input/chembl/chembl.tar.gz
        gene_ontology_human: input/target/go/goa_human.gaf.gz
        gene_ontology_rna: input/target/go/goa_human_rna.gaf.gz
        gene_ontology_rna_lookup: input/target/go/ensembl.tsv
        gene_ontology_eco_lookup: input/target/go/goa_human_eco.gpa.gz
        gnomad: input/target/gnomad/gnomad_constraint_metrics.tsv.bgz
        hallmarks: input/target/hallmarks/cosmic-hallmarks.tsv.gz
        hgnc: input/target/genenames/hgnc_complete_set.json
        hpa: input/target/hpa/subcellular_location.tsv.zip
        hpa_sl: input/target/hpa/subcellular_locations_ssl.tsv
        ncbi: input/target/ncbi/Homo_sapiens.gene_info.gz
        reactome_etl: output/reactome
        reactome_pathways: input/target/reactome/Ensembl2Reactome.txt
        uniprot: input/target/uniprot/uniprot.txt.gz
        uniprot_ssl: input/target/uniprot/uniprot-ssl.tsv.gz
      destination: output/target
  ##################################################################################################

  #: TRANSCRIPT STEP :#################################################################################
  transcript:
    - name: transform transcript
      transformer: transcript
      source:
        ensembl: input/ensembl/homo_sapiens
      destination: output/transcript
  ##################################################################################################

  #: TARGET_SAFETY_EVENT STEP :##########################################################################
  target_safety_event:
    - name: transform target_safety_event
      transformer: target_safety_event
      source:
        adverse_events: input/target/safety/adverse_effects.tsv
        safety_risks: input/target/safety/safety_risks.tsv
        brennan: input/target/safety/secondary_pharmacology.json
        toxcast: input/target/safety/toxcast.tsv
        aopwiki: input/target/safety/aopwiki.json
        pharmacogenetics: output/pharmacogenomics
        target: output/target
        diseases: output/disease
      destination: output/target_safety_event
  ##################################################################################################

  #: TARGET_TRACTABILITY STEP :#########################################################################
  target_tractability:
    - name: transform target_tractability
      transformer: target_tractability
      source:
        tractability: input/target/tractability/tractability.tsv
        target: output/target
      destination: output/target_tractability
  ##################################################################################################

  #: HOMOLOGY STEP :#####################################################################################
  homology:
    - name: transform homology
      transformer: homology
      source:
        species_dictionary: input/target/homologue/species_EnsemblVertebrates.txt
        homologies: input/target/homologue/homologies
        ensembl: input/ensembl
        target: output/target
      destination: output/homology
      settings:
        ensembl_species:
          - caenorhabditis_elegans
          - canis_lupus_familiaris
          - cavia_porcellus
          - danio_rerio
          - drosophila_melanogaster
          - macaca_mulatta
          - mus_musculus
          - oryctolagus_cuniculus
          - pan_troglodytes
          - rattus_norvegicus
          - sus_scrofa
          - xenopus_tropicalis
          - homo_sapiens
  ##################################################################################################

  #: TARGET_VIEW STEP :##################################################################################
  target_view:
    - name: transform target_view
      transformer: target_view
      source:
        target: output/target
        homology: output/homology
        target_tractability: output/target_tractability
        target_safety_event: output/target_safety_event
        chemical_probes: output/chemical_probe
        transcript: output/transcript
      destination: view/target_view
  ##################################################################################################

  #: TARGET_GENE_ESSENTIALITY STEP :################################################################
  target_gene_essentiality:
    - name: transform gene_essentiality
      transformer: gene_essentiality
      source:
        models: input/target/gene-essentiality/depmap/Model.csv
        essential_genes: input/target/gene-essentiality/depmap/CRISPRInferredCommonEssentials.csv
        gene_effects: input/target/gene-essentiality/depmap/CRISPRGeneEffect.csv
        gene_expression: input/target/gene-essentiality/depmap/OmicsExpressionTPMLogp1HumanProteinCodingGenes.csv
        mutation_hotspots: input/target/gene-essentiality/depmap/OmicsSomaticMutationsMatrixHotspot.csv
        mutation_damaging: input/target/gene-essentiality/depmap/OmicsSomaticMutationsMatrixDamaging.csv
        depmap_tissue_mapping: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv
        target: output/target
      destination: output/target_essentiality
      settings:
        keep_only_essentials: false
  ##################################################################################################

  #: PHARMACOGENETICS STEP :########################################################################
  pharmacogenetics:
    - name: transform pharmacogenetics
      transformer: pharmacogenetics
      source:
        clinpgx: input/pharmacogenetics/clinpgx_annotation.json.gz
        phenotypes: input/pharmacogenetics/phenotypes.json
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        chembl_molecule: intermediate/chembl_molecule
        drug_mechanism_of_action: output/drug_mechanism_of_action
      destination:
        associations: output/pharmacogenomics
        unparsed_texts: intermediate/pharmacogenetics/unparsed_texts.json
  ##################################################################################################

  #: EVIDENCE_PROJECT_SCORE STEP :##################################################################
  evidence_project_score:
    - name: transform evidence.project_score
      transformer: evidence.project_score
      source:
        gene_scores: input/evidence/project_score/mapped_diseases.tsv
        cell_types: input/evidence/project_score/cell_types.tsv
        cell_passport: input/evidence/cell_passport.csv.gz
        cell_line_mapping: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/crispr
        evidence: output/evidence_crispr
  ##################################################################################################

  #: EVIDENCE_GENE_BURDEN STEP :####################################################################
  evidence_gene_burden:
    - name: transform evidence.gene_burden
      transformer: evidence.gene_burden
      source:
        az_binary: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-binary
        az_quantitative: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-quantitative
        az_genes: input/evidence/gene_burden/astrazeneca/azphewas_com_genes_UK_Biobank_470k.csv
        az_phenotypes: input/evidence/gene_burden/astrazeneca/azphewas_com_phenotypes_UK_Biobank_470k.csv
        finngen: input/evidence/gene_burden/finngen/finngen.txt.gz
        finngen_phenotypes: input/evidence/gene_burden/finngen/phenos
        genebass: input/evidence/gene_burden/genebass
        cvdi: input/evidence/gene_burden/cvdi.xlsx
        genes_and_health: input/evidence/gene_burden/genes_health.xlsx
        brava: input/evidence/gene_burden/brava.xlsx
        curated_studies: input/ot_curation/gene_burden/curated_evidence.tsv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gene_burden
        evidence: output/evidence_gene_burden
      settings:
        finngen_release: R12
  ##################################################################################################

  #: EVIDENCE_CLINICAL_PRECEDENCE STEP :############################################################
  evidence_clinical_precedence:
    - name: transform evidence.clinical_precedence
      transformer: evidence.clinical_precedence
      source:
        clinical_report: output/clinical_report
        drug_mechanism_of_action: output/drug_mechanism_of_action
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/clinical_precedence
        evidence: output/evidence_clinical_precedence
        excluded_report: excluded/clinical_precedence_failed_report
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: EVIDENCE_CANCER_BIOMARKERS STEP :################################################################
  evidence_cancer_biomarkers:
    - name: transform evidence.cancer_biomarkers
      transformer: evidence.cancer_biomarkers
      source:
        associations: input/evidence/cancerbiomarkers/associations.tsv
        source: input/evidence/cancerbiomarkers/sources.jsonl
        biomarker_disease: input/evidence/cancerbiomarkers/diseases.jsonl
        drugs: output/drug_molecule
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/cancer_biomarkers
        evidence: output/evidence_cancer_biomarkers
  ##################################################################################################

  #: EVIDENCE_ORPHANET STEP :#######################################################################
  evidence_orphanet:
    - name: transform evidence.orphanet
      transformer: evidence.orphanet
      source:
        evidence: input/evidence/orphanet.xml
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/orphanet
        evidence: output/evidence_orphanet
  ##################################################################################################

  #: EVIDENCE_CLINGEN STEP :#######################################################################
  evidence_clingen:
    - name: transform evidence.clingen
      transformer: evidence.clingen
      source:
        evidence: input/evidence/clingen.csv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
      destination:
        failed_evidence: excluded/evidence/clingen
        evidence: output/evidence_clingen
  ##################################################################################################

  #: EVIDENCE_IMPC STEP :###########################################################################
  evidence_impc:
    - name: transform evidence.impc
      transformer: evidence.impc
      source:
        solr_gene_gene: input/impc/gene_gene.csv
        solr_ontology_ontology: input/impc/ontology_ontology.csv
        solr_mouse_model: input/impc/mouse_model.csv
        solr_disease: input/impc/disease.csv
        solr_disease_model_summary: input/impc/disease_model_summary.csv
        solr_ontology: input/impc/ontology.csv
        hgnc_gene_mappings: input/target/genenames/hgnc_complete_set.tsv
        mouse_gene_mappings: input/impc/MGI_Gene_Model_Coord.rpt
        mouse_pubmed_refs: input/impc/MGI_PhenoGenoMP.rpt
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/impc
        evidence: output/evidence_impc
      settings:
        score_cutoff: 41
  ##################################################################################################

  #: EVIDENCE PANEL APP STEP :#######################################################################
  evidence_panel_app:
    - name: transform evidence.panel_app
      transformer: evidence.panel_app
      source:
        evidence: input/evidence/panelapp/panel_app.tsv
        panels: input/evidence/panelapp/panels.jsonl
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/panel_app
        evidence: output/evidence_genomics_england
  ##################################################################################################

  #: EVIDENCE_CRISPR_SCREENS STEP :#################################################################
  evidence_crispr_screens:
    - name: transform evidence.crispr_screens
      transformer: evidence.crispr_screens
      source:
        screens: input/evidence/crispr_screens/brain_screens.json.gz
        studies_dir: input/evidence/crispr_screens/brain_studies
        disease_mapping: input/ot_curation/mappings/disease/brain_crispr_studies.tsv
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/crispr_screen
        evidence: output/evidence_crispr_screen
  ##################################################################################################

  #: EVIDENCE_GENE2PHENOTYPE STEP :#################################################################
  evidence_gene2phenotype:
    - name: transform evidence.gene2phenotype
      transformer: evidence.gene2phenotype
      source:
        cancer_panel: input/evidence/gene2phenotype/cancer.csv
        cardiac_panel: input/evidence/gene2phenotype/cardiac.csv
        developmental_panel: input/evidence/gene2phenotype/dd.csv
        ear_panel: input/evidence/gene2phenotype/ear.csv
        eye_panel: input/evidence/gene2phenotype/eye.csv
        skeletal_panel: input/evidence/gene2phenotype/skeletal.csv
        skin_panel: input/evidence/gene2phenotype/skin.csv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gene2phenotype
        evidence: output/evidence_gene2phenotype
  ##################################################################################################

  #: EVIDENCE_INTOGEN STEP :#######################################################################
  evidence_intogen:
    - name: transform evidence.intogen
      transformer: evidence.intogen
      source:
        genes: input/evidence/intogen/Compendium_Cancer_Genes.tsv
        cohorts: input/evidence/intogen/cohorts.tsv
        disease_mapping: input/ot_curation/mappings/disease/cancer2EFO_mappings.tsv
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/intogen
        evidence: output/evidence_intogen
  ##################################################################################################

  #: EVIDENCE PPP STEP :############################################################################
  evidence_ot_crispr:
    - name: transform evidence.ot_crispr
      transformer: evidence.ot_crispr
      source:
        study_table: input/evidence/ot_crispr/config.csv
        ot_crispr_data: input/evidence/ot_crispr/data
        target: output/target
        disease: output/disease
      destination:
        failed_evidence: excluded/evidence/ot_crispr
        evidence: output/evidence_ot_crispr
  ##################################################################################################

  #: ENHANCER_TO_GENE MIGRATION
  enhancer_to_gene:
    - name: transform enhancer to gene
      transformer: enhancer_to_gene
      source:
        enhancer_to_gene: input/encode/rE2G/datasets/*.bed.gz
        biosample_lut: input/encode/rE2G/biosample_mapping.csv
        target: output/target
        biosample: output/biosample
        chromosome: input/chromosome/chromosome.parquet
      destination:
        output: output/enhancer_to_gene
        excluded: excluded/enhancer_to_gene
      settings:
        min_valid_score: 0.6
        max_valid_score: 1.0
        invalid_qc_reasons:
          - UNRESOLVED_TARGET
          - UNKNOWN_BIOSAMPLE
          - SCORE_OUTSIDE_BOUNDS
          - UNKNOWN_INTERVAL_TYPE
          - AMBIGUOUS_SCORE
          - INVALID_CHROMOSOME
          - INVALID_RANGE
          - AMBIGUOUS_INTERVAL_TYPE
          - TRANS_CHROMOSOMAL_EFFECT

  #: ONTOMA STEPS :##################################################################################
  ontoma:
    - name: transform disease lookup tables for ontoma
      transformer: ontoma_lut.ontoma
      source:
        disease_index: output/disease
        ot_disease_curation: input/ontoma/ot_disease_curation.tsv
        eva_clinvar: input/ontoma/eva_clinvar.txt
        clinvar_xrefs: input/ontoma/clinvar_xrefs.txt
      destination:
        disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet

  # Split from ontoma so that the disease tables don't wait on target: pts_literature_ontoma needs
  # pts_target, and pts_target_safety_event → pts_pharmacogenetics needs pts_ontoma.
  literature_ontoma:
    - name: transform disease target drug lookup table for ontoma
      transformer: ontoma_lut.literature_ontoma
      source:
        disease_index: output/disease
        ot_disease_curation: input/ontoma/ot_disease_curation.tsv
        eva_clinvar: input/ontoma/eva_clinvar.txt
        clinvar_xrefs: input/ontoma/clinvar_xrefs.txt
        target_index: output/target
        drug_index: output/drug_molecule
      destination:
        disease_target_drug_label_lut: intermediate/ontoma/disease_target_drug_label_lookup_table.parquet
  ##################################################################################################

  #: DRUG_WARNING STEP :############################################################################
  drug_warning:
    - name: transform drug_warning
      transformer: drug_warning
      source: input/chembl/chembl.tar.gz
      destination: output/drug_warning
  ##################################################################################################

  #: DRUG_MECHANISM_OF_ACTION STEP :###############################################################
  drug_mechanism_of_action:
    - name: transform drug_mechanism_of_action
      transformer: drug_mechanism_of_action
      source:
        chembl: input/chembl/chembl.tar.gz
        target: output/target
      destination: output/drug_mechanism_of_action
  ##################################################################################################

  #: CHEMBL_MOLECULE STEP :#########################################################################
  chembl_molecule:
    - name: transform chembl_molecule
      transformer: chembl_molecule
      source:
        chembl: input/chembl/chembl.tar.gz
        drugbank: input/drug/drugbank.csv.gz
        aact_trial_extraction: input/clinical_report/aact_trial_extraction
      destination: intermediate/chembl_molecule
  ##################################################################################################

  #: DRUG_MOLECULE STEP :##########################################################################
  drug_molecule:
    - name: transform drug_molecule
      transformer: drug_molecule
      source:
        molecule: intermediate/chembl_molecule
        chemical_probes: output/chemical_probe
        mechanism_of_action: output/drug_mechanism_of_action
        clinical_report: output/clinical_report
        disease: output/disease
      destination:
        output: output/drug_molecule
        excluded: excluded/drug_molecule_failed_report
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
  ##################################################################################################

  #: AACT_TRIAL_EXTRACTION STEP :##################################################################
  # Not part of a release. The `clinical_staging` DAG runs this step with
  # release_uri pointed at gs://clinical_staging/aact/${staging_version}, so every path below
  # resolves inside that snapshot, except for the record store
  aact_trial_extraction:
    - name: transform aact_trial_extraction
      transformer: clinical_mira.aact_trial_extraction
      source:
        aact: input/aact.zip
      destination:
        extraction: extraction
      settings:
        store_uri: gs://clinical_staging/store
        # the AACT version, prefixing the store runs this step writes
        snapshot: '{{snapshot}}'
        model: gpt-5-nano-2025-08-07
        service_tier: auto
        concurrency: 75
        shard_size: 200
  ##################################################################################################

  #: AACT_TRIAL_MODELS STEP :######################################################################
  # Not part of a release. Runs in the `clinical_staging` DAG, with
  # release_uri pointed at gs://clinical_staging/aact/${staging_version}; the record stores sit outside it
  aact_trial_models:
    - name: transform aact_trial_models
      transformer: clinical_mira.aact_trial_models
      source:
        aact: input/aact.zip
        extraction: extraction
        models: input/models
      destination:
        stop_reasons: stop_reasons
      settings:
        store_uri: gs://clinical_staging/store
        snapshot: '{{snapshot}}'
  ##################################################################################################

  #: CLINICAL_MINING_MODELS STEP :#################################################################
  # Not part of a release. Runs with release_uri pointed at
  # gs://clinical_staging/clinical_mining/${staging_version}; the record stores sit outside it
  clinical_mining_models:
    - name: transform clinical_mining_models
      transformer: clinical_mira.clinical_mining_models
      source:
        ttd: input/ttd.txt
        ema: input/ema_medicines.xlsx
        pmda: input/pmda.pdf
        models: input/models
      destination:
        drug_entities: drug_entities
        disease_entities: disease_entities
      settings:
        store_uri: gs://clinical_staging/store
        snapshot: '{{snapshot}}'
  ##################################################################################################

  #: CLINICAL_REPORT STEP :#######################################################################
  clinical_report:
    - name: transform clinical_report
      transformer: clinical_report
      source:
        chembl_molecule: intermediate/chembl_molecule
        disease: output/disease
        pmda: input/clinical_report/pmda.pdf
        ttd: input/clinical_report/ttd.txt
        ema: input/clinical_report/ema_medicines.xlsx
        chembl: input/chembl/chembl.tar.gz
        aact: input/clinical_report/aact.zip
        chembl_curation: input/clinical_report/chembl_mapping.parquet
        trial_extraction: input/clinical_report/aact_trial_extraction
        aact_stop_reasons: input/clinical_report/aact_stop_reasons
        aact_drug_entities: input/clinical_report/aact_drug_entities
        mining_drug_entities: input/clinical_report/mining_drug_entities
        mining_disease_entities: input/clinical_report/mining_disease_entities
      destination:
        output: output/clinical_report
  ##################################################################################################

  #: CLINICAL_INDICATION STEP :####################################################################
  clinical_indication:
    - name: transform clinical_indication
      source: output/clinical_report
      destination:
        output: output/clinical_indication
        excluded: excluded/clinical_indication_failed_report
      transformer: clinical_indication
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: CLINICAL_TARGET STEP :#########################################################################
  clinical_target:
    - name: transform clinical_target
      source:
        clinical_report: output/clinical_report
        drug_mechanism_of_action: output/drug_mechanism_of_action
      destination:
        output: output/clinical_target
        excluded: excluded/clinical_target_failed_report
      transformer: clinical_target
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: TARGET_PRIORITISATION STEP :###################################################################
  target_prioritisation:
    - name: transform target_prioritisation
      transformer: target_prioritisation
      source:
        targets: output/target
        tractability: output/target_tractability
        target_safety_event: output/target_safety_event
        chemical_probes: output/chemical_probe
        homology: output/homology
        mouse_phenotypes: output/mouse_phenotype
        molecule: output/drug_molecule
        mechanism_of_action: output/drug_mechanism_of_action
        baseline_expression: output/baseline_expression
        uniprot_slterms: input/target_prioritisation/uniprot_locations.tsv.gz
        mouse_pheno_scores: input/target_prioritisation/mouse_pheno_scores.csv
      destination: output/target_prioritisation
  ##################################################################################################

  #: ASSOCIATION STEP :#############################################################################
  association:
    - name: pyspark association
      pyspark: association
      source:
        evidence: output/evidence_*
        disease: output/disease
      destination:
        overall_direct: output/association_overall_direct
        overall_indirect: output/association_overall_indirect
        by_datasource_direct: output/association_by_datasource_direct
        by_datasource_indirect: output/association_by_datasource_indirect
        by_datatype_direct: output/association_by_datatype_direct
        by_datatype_indirect: output/association_by_datatype_indirect
        temporary: intermediate/indirect_association_intermediate
      properties:
        spark.driver.memory: '12g'
      settings:
        partition_count:
          overall_direct: 14
          overall_indirect: 39
          by_datasource_direct: 14
          by_datasource_indirect: 43
          by_datatype_direct: 15
          by_datatype_indirect: 41
        novelty_scale: 2 # 2 for long tailed slow decays
        novelty_shift: 3 # 3 for long tailed slow decays
        novelty_window: 10
        datasource_weights:
          - datasourceId: cancer_biomarkers
            weight: 0.5
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: cancer_gene_census
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: clingen
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: clinical_precedence
            weight: 1.0
            datatypeId: known_drug
            toPropagate: true
          - datasourceId: crispr
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: crispr_screen
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: encore
            weight: 0.5
            datatypeId: ot_partner
            toPropagate: true
          - datasourceId: europepmc
            weight: 0.2
            datatypeId: literature
            toPropagate: true
          - datasourceId: eva
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: eva_somatic
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: expression_atlas
            weight: 0.2
            datatypeId: rna_expression
            toPropagate: false
          - datasourceId: gene2phenotype
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: gene_burden
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: genomics_england
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: gwas_credible_sets
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: impc
            weight: 0.2
            datatypeId: animal_model
            toPropagate: true
          - datasourceId: intogen
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: orphanet
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: ot_crispr
            weight: 0.5
            datatypeId: ot_partner
            toPropagate: true
          - datasourceId: ot_crispr_validation
            weight: 0.5
            datatypeId: ot_validation_lab
            toPropagate: true
          - datasourceId: reactome
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: uniprot_literature
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: uniprot_variants
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
  ##################################################################################################

  #: EVIDENCE POST PROCESS STEP :###################################################################
  evidence_expression_atlas:
    - name: transform evidence.expression_atlas
      transformer: evidence.expression_atlas
      source:
        evidence: input/evidence/atlas.parquet
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/expression_atlas
        evidence: output/evidence_expression_atlas

  evidence_clinvar:
    - name: transform evidence.clinvar
      transformer: evidence.clinvar
      source:
        evidence: input/evidence/eva.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/eva
        evidence: output/evidence_eva

  evidence_clinvar_somatic:
    - name: transform evidence.clinvar_somatic
      transformer: evidence.clinvar_somatic
      source:
        evidence: input/evidence/eva.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/eva_somatic
        evidence: output/evidence_eva_somatic

  evidence_uniprot_variants:
    - name: transform evidence.uniprot_variants
      transformer: evidence.uniprot_variants
      source:
        evidence: input/evidence/uniprot_variants.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/uniprot_variants
        evidence: output/evidence_uniprot_variants

  evidence_uniprot_literature:
    - name: transform evidence.uniprot_literature
      transformer: evidence.uniprot_literature
      source:
        evidence: input/evidence/uniprot_literature.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/uniprot_literature
        evidence: output/evidence_uniprot_literature

  evidence_reactome:
    - name: transform evidence.reactome
      transformer: evidence.reactome
      source:
        evidence: input/evidence/reactome.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/reactome
        evidence: output/evidence_reactome

  evidence_europepmc:
    - name: transform evidence.europepmc
      transformer: evidence.europepmc
      source:
        cooccurrence: intermediate/literature/cooccurrence
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/europepmc
        evidence: output/evidence_europepmc

  evidence_cosmic:
    - name: transform evidence.cosmic
      transformer: evidence.cosmic
      source:
        evidence: input/evidence/cosmic.json.gz
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/cancer_gene_census
        evidence: output/evidence_cancer_gene_census

  evidence_encore:
    - name: transform evidence.encore
      transformer: evidence.encore
      source:
        evidence: input/evidence/encore
        target: output/target
        disease: output/disease
      destination:
        failed_evidence: excluded/evidence/encore
        evidence: output/evidence_encore

  evidence_validation_lab:
    - name: transform evidence.validation_lab
      transformer: evidence.validation_lab
      source:
        evidence: input/evidence/validation_lab.json.gz
        target: output/target
        disease: output/disease
      destination:
        failed_evidence: excluded/evidence/ot_crispr_validation
        evidence: output/evidence_ot_crispr_validation
  ##################################################################################################

  #: ASSOCIATION_OTF STEP :#########################################################################
  association_otf:
    - name: pyspark association_otf
      pyspark: association_otf
      properties:
        spark.driver.memory: '20g'
      source:
        disease: output/disease
        evidence: output/evidence_*
        target: output/target
        tractability: output/target_tractability
        association_direct: output/association_overall_direct
        association_indirect: output/association_overall_indirect
      destination: view/association_otf
  ##################################################################################################

  #: L2G STEP :#####################################################################################
  l2g:
    - name: transform l2g_train
      transformer: l2g_train
      source:
        feature_matrix: intermediate/l2g_feature_matrix
        credible_set: output/credible_set
        # built by gentropy_l2g_training_set from the effector gene list
        gold_standard: intermediate/l2g_training_set
        predefined_test: input/l2g/predefined_test_for_split
      destination:
        model: etc/model/locus_to_gene_model/classifier.skops
        background: etc/model/locus_to_gene_model/shap_background.parquet
        metrics: etc/model/locus_to_gene_model/metrics.json
        train_split: intermediate/l2g_train_split
        test_split: intermediate/l2g_test_split
        split_stats: intermediate/l2g_train_test_split_stats.json
      settings:
        # fitted order; also the order of the output `features` array
        features_list: &l2g_features
          - eQtlColocClppMaximum
          - pQtlColocClppMaximum
          - sQtlColocClppMaximum
          - eQtlColocH4Maximum
          - pQtlColocH4Maximum
          - sQtlColocH4Maximum
          - eQtlColocClppMaximumNeighbourhood
          - pQtlColocClppMaximumNeighbourhood
          - sQtlColocClppMaximumNeighbourhood
          - eQtlColocH4MaximumNeighbourhood
          - pQtlColocH4MaximumNeighbourhood
          - sQtlColocH4MaximumNeighbourhood
          - distanceSentinelFootprint
          - distanceSentinelFootprintNeighbourhood
          - distanceFootprintMean
          - distanceFootprintMeanNeighbourhood
          - distanceTssMean
          - distanceTssMeanNeighbourhood
          - distanceSentinelTss
          - distanceSentinelTssNeighbourhood
          - vepMaximum
          - vepMaximumNeighbourhood
          - vepMean
          - vepMeanNeighbourhood
          - e2gMean
          - e2gMeanNeighbourhood
          - geneCount500kb
          - proteinGeneCount500kb
          - credibleSetConfidence
          - transPQtlColocH4Maximum
          - transPQtlColocH4MaximumNeighbourhood
        # Pinned in #123.
        hyperparameters:
          objective: binary:logistic
          eval_metric: aucpr
          random_state: 777
          n_estimators: 200
          max_depth: 4
          min_child_weight: 10
          eta: 0.025
          subsample: 0.8
          colsample_bytree: 0.8
          reg_alpha: 1
          reg_lambda: 1.0
          scale_pos_weight: 0.8
          gamma: 0
          max_delta_step: 1
        train_on_full_dataset: true
        shap_background_size: 100
        shap_background_seed: 42
    - name: transform l2g_predict
      requires:
        - transform l2g_train
      transformer: l2g_predict
      source:
        feature_matrix: intermediate/l2g_feature_matrix
        credible_set: output/credible_set
        model: etc/model/locus_to_gene_model/classifier.skops
        background: etc/model/locus_to_gene_model/shap_background.parquet
      destination: output/l2g_prediction
      settings:
        features_list: *l2g_features
        l2g_threshold: &l2g_threshold 0.05
        explain_predictions: true
        # shap_background_size is set on l2g_train only

  #: EVIDENCE_GWAS_CREDIBLE_SETS STEP :#############################################################
  evidence_gwas_credible_sets:
    - name: transform evidence.gwas_credible_sets
      transformer: evidence.gwas_credible_sets
      source:
        l2g_predictions: output/l2g_prediction
        credible_set: output/credible_set
        study_index: output/study
        target: output/target
        disease: output/disease
        literature: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gwas_credible_sets
        evidence: output/evidence_gwas_credible_sets
      settings:
        locus_to_gene_threshold: *l2g_threshold

  #: VEP_VIEW STEP :################################################################################
  vep_view:
    - name: transform vep view
      transformer: vep_view
      source:
        credible_set: output/credible_set
        credible_set_variant: output/credible_set_variant
        study_table: output/study
        l2g_table: output/l2g_prediction
        biosample: output/biosample
      destination:
        vep_data_header: intermediate/vep_view/vep_view_header.tsv
        vep_data_all: intermediate/vep_view/vep_view
        vep_data_gwas: intermediate/vep_view/vep_view_gwas
        vep_data_qtl: intermediate/vep_view/vep_view_qtl
  ##################################################################################################

  #: ASSOCIATION_TIMESERIES_VIEW STEP :#############################################################
  association_timeseries_view:
    - name: pyspark timeseries view
      pyspark: association_timeseries_view
      source:
        association_overall_direct: output/association_overall_direct
        association_overall_indirect: output/association_overall_indirect
        association_by_datasource_direct: output/association_by_datasource_direct
        association_by_datasource_indirect: output/association_by_datasource_indirect
      destination: view/association_timeseries
      properties:
        spark.driver.memory: 12g
  ##################################################################################################

  #: GENE_SETS STEP :###############################################################################
  gene_sets:
    - name: transform gene_sets
      transformer: gene_sets
      source:
        targets: output/target
        go: output/go
        reactome: output/reactome
        chembl: input/chembl/chembl.tar.gz
      destination: output/gene_sets
  ##################################################################################################

  #: SEARCH_FACET STEP :############################################################################
  search_facet:
    - name: transform search_facet
      transformer: search_facet
      source:
        diseases: output/disease
        targets: output/target
        gene_sets: output/gene_sets
        tractability: output/target_tractability
      destination:
        targets: view/search_facet_target
        diseases: view/search_facet_disease
  ##################################################################################################

  #: INTERACTION STEP :#############################################################################
  interaction:
    - name: transform interaction
      transformer: interaction
      source:
        targets: output/target
        rnacentral: input/interaction/rna_central_ensembl.tsv
        humanmapping: input/interaction/HUMAN_9606_idmapping.dat.gz
        ensproteins: input/interaction/Homo_sapiens.GRCh38.chr.gtf.gz
        intact: input/interaction/intact-interactors.json
        string_detailed: input/interaction/string-links-detailed.txt.gz
        string_full: input/interaction/string-links-full.txt.gz
      destination:
        interactions: output/interaction
        interactions_evidence: output/interaction_evidence
        interactions_unmatched: excluded/interaction
      settings:
        scorethreshold: 0
        string_version: ${string_version}
  ##################################################################################################

  #: RELEASE_METRICS STEP :#########################################################################
  release_metrics:
    - name: transform release_metrics
      transformer: release_metrics
      source: {}
      destination:
        parquet: metrics.parquet
      settings:
        ot_release: '{{release_name}}'
        metric_scopes:
          - /output/*/
          - /excluded/evidence/*/
        rich_dataset_whitelist:
          - /output/evidence_*
          - /excluded/evidence/*
          - /output/association_by_datasource_direct
          - /output/association_by_datasource_indirect
          - /output/disease
          - /output/target
          - /output/drug_molecule
          - /output/study
          - /output/clinical_report
  ##################################################################################################

  #: SEARCH STEP :##################################################################################
  search:
    - name: transform search
      transformer: search
      source:
        association: output/association_overall_indirect
        drug: output/drug_molecule
        evidence: output/evidence_*
        indication: output/clinical_indication
        mechanism: output/drug_mechanism_of_action
        target: output/target
        credible_sets: output/credible_set
        disease: output/disease
        disease_hpo: output/disease_phenotype
        hpo: output/disease_hpo
        studies: output/study
        variants: output/variant
      destination:
        diseases: view/search_disease
        targets: view/search_target
        drugs: view/search_drug
        variants: view/search_variant
        studies: view/search_study
  ##################################################################################################

  #: SEARCH_EBI STEP :##############################################################################
  search_ebi:
    - name: transform search_ebi
      transformer: search_ebi
      source:
        disease: output/disease
        target: output/target
        association: output/association_overall_direct
        evidence: output/evidence_gwas_credible_sets
      destination:
        associations: view/search_ebi_associations
        evidence: view/search_ebi_evidence
  ##################################################################################################

  #: LITERATURE STEPS (run on pts_literature cluster) :#############################################
  literature_publication_match:
    - name: pyspark literature publication match
      pyspark: literature_publication_match
      source:
        pub_id_lut: input/literature/PMID_PMCID_DOI.csv.gz
        epmc_publication: gs://otar025-epmc/ml02
        ontoma_disease_target_drug_label_lut: intermediate/ontoma/disease_target_drug_label_lookup_table.parquet
      destination:
        match_valid: intermediate/literature/match
        match_failed: excluded/literature/match
      settings:
        date_prefix: ''
        repartition: 25000
        label_mapping_partitions: 768
        match_valid_coalesce: 600
        match_failed_coalesce: 530
      properties:
        # Per-step overrides for the heavy publication_match step. Held-executor
        # pin (dynamicAllocation.*) survives the autoscaler downscale issue
        # observed in PTS run-015. AQE coalesce + skew handling and the EPMC
        # file partition cap (32MB) were validated on runs 013-016.
        'spark.sql.shuffle.partitions': '15000'
        'spark.sql.adaptive.skewJoin.skewedPartitionFactor': '2'
        'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes': '64MB'
        'spark.sql.adaptive.advisoryPartitionSizeInBytes': '268435456'
        'spark.sql.adaptive.coalescePartitions.parallelismFirst': 'false'
        'spark.sql.files.maxPartitionBytes': '33554432'
        'spark.dynamicAllocation.executorIdleTimeout': '7200s'
        'spark.dynamicAllocation.shuffleTracking.enabled': 'true'
        'spark.dynamicAllocation.shuffleTracking.timeout': '7200s'
        'spark.executorEnv.POLARS_MAX_THREADS': '1'

  literature_entity_lut:
    - name: pyspark literature entity lut
      pyspark: literature_entity_lut
      source:
        matches: intermediate/literature/match
      destination:
        literature_entity_lut: output/literature_entity_lut
      settings:
        partition_count: 35
      properties:
        'spark.sql.shuffle.partitions': '10000'

  literature_embedding:
    - name: pyspark literature embedding
      pyspark: literature_embedding
      source:
        matches: intermediate/literature/match
      destination:
        model: etc/model/w2v_model
      properties:
        'spark.sql.shuffle.partitions': '15000'

  literature_vector:
    - name: pyspark literature vector
      pyspark: literature_vector
      source:
        model: etc/model/w2v_model
      destination:
        vectors: output/literature_vector

  literature_cooccurrence:
    - name: pyspark literature cooccurrence
      pyspark: literature_cooccurrence
      source:
        match: intermediate/literature/match
      destination:
        cooccurrence: intermediate/literature/cooccurrence
      settings:
        cooccurrence_coalesce: 40
      properties:
        'spark.sql.shuffle.partitions': '15000'
        'spark.sql.adaptive.skewJoin.skewedPartitionFactor': '2'
        'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes': '64MB'
  ##################################################################################################