# raven.eval — scoring contract # Every published number is produced under exactly this config. Changing any # field here changes the numbers; the config is committed so third parties see # the precise rules our DER/WER values were computed under. # Which of the top-level blocks below are metric contracts. Named explicitly, # because deriving it as "everything that is not `datasets`" turns any new # top-level key — publication rules, provenance notes — into a metric that the # contract test then demands an implementation for. The list is checked in both # directions against raven_eval_core.SCORED_METRICS. metrics: [der, wer, bleu, entity] der: scorer_primary: pyannote.metrics # Independent cross-check against nryant/dscore (the md-eval reference wrapper), # `make dscore-check` / raven_diar.dscore_check. Optional and NOT in CI: dscore # is a script repo, not a pip package, so it is gated on a local DSCORE_DIR # checkout and skips cleanly without one. # status: RUN on 2026-09-04 against dscore @ e02f949 — 15 (gold, hyp) pairs # across 4 diarizers and 3 datasets, both collars, 30 comparisons, # worst |Δ| = 0.030 pp. This line said "WIRED" for weeks while the check had # never executed once and carried a collar bug that would have failed it. # note: dscore's --collar is applied per side, pyannote's is the total window # width, so `dscore --collar X` == `pyannote collar 2X`. The conversion lives # in DSCORE_COLLAR_FACTOR and is asserted by tests/test_dscore_check.py. # DER is reported TWICE — the two conventions are NOT comparable, so we publish both: variants: - name: full # pyannote / DIHARD convention collar: 0.0 skip_overlap: false - name: classic # NIST / CALLHOME convention (Raven's historical value) collar: 0.25 skip_overlap: false # WHICH configuration a model with more than one was measured under. A # streaming diarizer does not have one DER, it has one per latency setting: # on German CALLHOME the same Sortformer v2 checkpoint reads 8.98 at 30.4 s of # input-buffer latency and 9.82 at 1.04 s. That is 0.84 pp — seventeen times # the +-0.05 pp tolerance every number here is reproduced to, and it is the # whole explanation of our disagreement with the ETH benchmark's 9.6. So the # operating point is part of the contract, not a runtime detail. # The effective value is also written into every run's summary.json (and thus # into the artifact), so a run at a non-default setting cannot be promoted as # if it were this one. Presets are named in raven_diar/adapters/sortformer.py # (LATENCY_PRESETS), transcribed from NVIDIA's model card and pinned by test. model_configuration: sortformer-streaming-4spk-v2: latency_preset: very-high-latency # 30.4 s input buffer, NVIDIA's lowest-DER row speaker_mapping: hungarian # optimal, via scipy.linear_sum_assignment vad: automatic # no oracle VAD, no oracle #speakers, no per-dataset tuning # HOW the per-file scores become one dataset number. Pinned here because it is # not a formatting choice: on our own German CALLHOME row the two conventions # read 11.41 and 11.07 — 0.334 pp apart, nearly seven times the ±0.05 pp # tolerance every number on this page is reproduced to. A published DER that # does not name its aggregation is as under-specified as one without a collar. # corpus Σ(miss+fa+conf) / Σ(scored reference speech) over all files. # Weights a file by how much speech it contributes, so a 50-minute # meeting outweighs a 30-second clip. The CALLHOME / DIHARD # convention and the one the pyannote model cards report. # file_mean unweighted mean of per-file DERs. Weights every file alike. The # convention the ETH diarization benchmark uses (arXiv 2509.26177, # Table 3 caption: "averaging all samples … and averaging over # them") — and the reason our 11.41 and their 11.1 are the same # measurement rather than a discrepancy. # Both are computed and both are committed to every expected.json # (der_full / der_classic vs der_full_filemean / der_classic_filemean). # `primary` is the one the headline table prints. aggregation: primary: corpus also_reported: [file_mean] # The per-file scores the aggregations are built from are themselves committed # (raven_diar/score.py -> FileScore), which is what makes the four claims below # checkable instead of asserted: # * a bootstrap confidence interval over files (raven_diar.analysis) # * DER split by REFERENCE speaker count # * the reference corpus overlap fraction # * speaker-aware boundary offsets + the length regime the misses sit in # `make analyse ARTIFACT=artifacts//` recomputes all of them from # the committed RTTMs, seeded, on a laptop. uncertainty: method: percentile bootstrap over files # the sampling unit the corpora were drawn in resamples: 10000 seed: 20260903 # a CI that moves between runs is not a number confidence: 0.95 comparisons: paired # two models resampled on the SAME files # Hosted diarizers return speaker-labelled WORDS, not turns. Folding words # into turns bridges silences up to this many seconds; every hosted adapter # uses the one shared implementation (raven_diar/adapters/aggregate.py) so a # DER gap between two vendors measures the models and not two private # foldings. It is declared here because it is a PUBLISHED quantity: measured # on 10 CALLHOME-de files with Deepgram, DER moves about 1 pp across a 4x # sweep of it — twenty times the 0.05 pp reproduction tolerance. # gap 0.25 -> 28.96 / 21.22 (full / classic collar) # gap 0.50 -> 28.33 / 20.30 <- published # gap 1.00 -> 29.54 / 21.07 # The trade is the expected one: a wider gap misses less and false-alarms # more. 0.5 is not tuned, it is the middle of a shallow curve — but a reader # who does not know the value cannot reproduce the number, which is why it # stopped being an implementation detail. turn_gap_merge_s: 0.5 wer: scorer: jiwer # The published WER numbers are produced by the flozi-strict method: # unidecode transliteration + alpha2digit (number WORDS -> digits) + # jiwer wer_standardize transforms, corpus aggregation, raw-text CER. # Alignment is PER UTTERANCE — the one deliberate divergence from flozi's code, # which concatenates the subset into a single sentence first and so lets an # error at one clip's end cancel against one at the next clip's start. That read # 0.26-0.85 pp low on VoxPopuli and the Swiss sets and identical on flozi's own # subsets (raven_eval_core/flozi_wer.py docstring). Changed 2026-09-19. # This is NOT the strict-de lens (that one lowercases inline, runs num2words in the # opposite digits->words direction, and strips fillers) — see the reconciliation # note in the header of scripts/verify.py for why the two do not interchange. normalization: flozi-strict alignment: per-utterance # Σedits/Σref_words over per-utterance alignments; decomposes exactly for the interval below also_reported: [permissive, verbatim, cer, ier] # diagnostic lenses in raven_eval_core.wer, not the published number # Every published WER carries a 95 % interval, committed to expected.json and # re-computed by `make verify`. The utterance is the sampling unit because it # is the unit the test sets were drawn in; each resample re-aggregates # Σedits/Σref_words, the published estimator, never a mean of utterance WERs. # Same settings as der.uncertainty, same resampler (raven_eval_core.bootstrap). uncertainty: method: percentile bootstrap over utterances resamples: 10000 seed: 20260903 confidence: 0.95 comparisons: paired # two models resampled on the SAME utterances, matched by sample_id bleu: scorer: sacrebleu # rejected: nltk.translate.bleu_score — no pinned tokenizer, so the single most score-moving convention would sit outside this contract # BLEU is reported for TRANSLATION-SHAPED corpora only: Swiss-German dialect is # SPOKEN in dialect and TRANSCRIBED in standard German, so the reference is a # translation, not a transcript. WER counts every legitimate lexical choice # ("gäll" -> "nicht wahr") as a substitution and mismeasures that shape; such a # dataset is scored bleu+wer, and the BLEU is the headline of the pair. # "BLEU" alone is not a number (Post 2018, "A Call for Clarity in Reporting BLEU # Scores") — every convention that moves it is pinned here and mirrored by the # constants in raven_eval_core/bleu.py. tests/test_metric_contract.py fails the # build if the two ever disagree, in either direction. variants: - name: published # the number we publish tokenize: 13a # mteval-v13a, the WMT default lowercase: false # case-sensitive (signature reads case:mixed) — German capitalizes nouns smooth_method: exp # sacrebleu default exponential smoothing, pinned rather than inherited effective_order: false # all 4 orders at corpus level max_ngram_order: 4 aggregation: corpus # Σ n-gram stats over the subset, NEVER a mean of per-sentence BLEUs signature: sacrebleu # every published BLEU carries its sacrebleu signature (e.g. nrefs:1|case:mixed|eff:no|tok:13a|smooth:exp|version:2.6.0) — that string, not the word "BLEU", is what makes it comparable also_reported: [sentence] # raven_eval_core.bleu.sentence_bleu_diagnostic — per-row diagnostic under effective_order=True, not the published number entity: # Numeric entity hit rate: the share of the reference's numbers — counts, # amounts, percentages, dates, times — that the transcript reproduces exactly. # A wrong digit is the costliest error in a meeting transcript and one # substitution among hundreds to WER, so it is scored on its own (as # KugelAudio's open German TTS benchmark protocol separates entity correctness # from CER). Scored on the same predictions_*.jsonl as WER; committed to the # same expected.json and re-derived by `make verify`. # Definition in full: raven_eval_core/entities.py docstring. scorer: raven_eval_core.entities # flozi-strict (alpha2digit turns number words into digits) read BEFORE its # final punctuation strip, which would fuse "3,5" into "35" and "2-3" into # "23" and so credit a wrong number. Reference and hypothesis are normalized # independently; a hit is exact string equality. normalization: flozi-strict-before-punctuation-strip entity: "[0-9]+([.,:][0-9]+)*, German thousands dots dropped" matching: multiset-per-utterance # an entity twice in the reference must appear twice in the hypothesis aggregation: corpus # Σhits / Σreference entities; utterances without an entity contribute nothing # Same resampler and settings as wer.uncertainty, but the unit is an utterance # that HOLDS an entity: resampling entity-free utterances would let a resample # draw none of the few that matter and score it 0 %. uncertainty: method: percentile bootstrap over utterances with at least one reference entity resamples: 10000 seed: 20260903 confidence: 0.95 comparisons: paired # two models resampled on the SAME utterances, matched by sample_id # Public datasets the Tier-2 full-rerun scores against. Audio is fetched by the # harness itself (HF at a pinned revision / the official mirror); we never # redistribute restricted audio. Every id below resolves to a loader in this # repo, and every loader in this repo appears below — tests/test_dataset_contract.py # fails the build in both directions, so this list cannot drift from the code. # # Every entry says WHAT is spoken in three separate fields, because one label # cannot carry all three without lying about one of them: # language_tag a valid BCP 47 tag. A region subtag is an ISO 3166-1 COUNTRY, # never a federal state or canton: "de-BY" is German as spoken # in Belarus, not Bavarian, and "gsw-BE" is Belgium, not Bern. # Bavarian is its own language subtag (bar), Swiss German gsw. # variety_label the variety and register in words, for a human reader. # locality where the variety is from, as the SOURCE documents it; null # where the source documents nothing or the question does not # apply. "Swiss German" is not one evaluation category — a # corpus spanning localities is reported per locality before # anything is aggregated. # tests/test_dataset_contract.py validates the three fields on every entry. datasets: der: - id: callhome-de # talkbank/callhome, deu config — free, gold, German (2-spk telephone) hf: talkbank/callhome config: deu license: TalkBank (cite-to-use) language_tag: de variety_label: "German (telephone conversation)" locality: null # no locality claim: the gold labels carry none - id: ami # CC BY 4.0 — 4-speaker meetings license: CC-BY-4.0 language_tag: en variety_label: "English (meetings)" locality: null # no locality claim: the gold labels carry none - id: voxconverse # dev split; labels CC BY 4.0; audio = YouTube owners license: CC-BY-4.0 (labels) language_tag: en variety_label: "English (multi-speaker YouTube video)" locality: null - id: voxconverse-test # same corpus, TEST split — the split pyannote publishes its DER on license: CC-BY-4.0 (labels) language_tag: en variety_label: "English (multi-speaker YouTube video)" locality: null wer: - id: avemio-german-mixed-test # avemio re-upload of the flozi00/asr-german-mixed TEST split (CV + MLS German) hf: avemio/ASR-GERMAN-MIXED-TEST license: CC-BY-4.0 (strictest component; CV CC0 + MLS CC-BY-4.0) language_tag: de variety_label: "Standard German (read: Common Voice + MLS)" locality: null - id: fleurs # google/fleurs config de_de, split test — read parallel sentences hf: google/fleurs config: de_de license: CC-BY-4.0 language_tag: de variety_label: "Standard German (read)" locality: null # de_de is FLEURS' locale label, not a documented speaker origin - id: mls-de # facebook/multilingual_librispeech config german, split test — LibriVox audiobooks hf: facebook/multilingual_librispeech config: german license: CC-BY-4.0 language_tag: de variety_label: "Standard German (read audiobooks)" locality: null - id: voxpopuli-de # facebook/voxpopuli config de, split test — EU-Parliament plenary speech hf: facebook/voxpopuli config: de license: CC0-1.0 language_tag: de variety_label: "Standard German (EU Parliament speech)" locality: null - id: german-mixed # avemio / flozi00 asr-german-mixed-evals (Tuda-De CC-BY-4.0, MLS CC-BY-4.0, CV CC0) hf: flozi00/asr-german-mixed-evals license: CC-BY-4.0 / CC0-1.0 (per subset) language_tag: de variety_label: "Standard German (read: Tuda-De, MLS, Common Voice)" locality: null # Swiss German dialect corpora. Translation-shaped — Swiss German spoken, # Standard German written — so `metric: bleu+wer`. Each is reported on its # own: there is no aggregate spanning dialects and no dialect contribution to # any overall average. - id: spc-test # Swiss Parliaments Corpus test, Bernese cantonal parliament hf: i4ds/SPC_test license: MIT (inferred from upstream FHNW SPC; the HF card carries no license tag) language_tag: gsw-CH variety_label: "Swiss German (Bernese)" locality: "Canton of Bern" # Bernese cantonal parliament metric: bleu+wer acquisition: two test parquet shards by URL, sha256-verified (raven_asr/datasets/spc_test.py) - id: fhnw-all-dialects # FHNW All Swiss German Dialects Test Set, SwissText 2021 task 3 license: MIT language_tag: gsw-CH variety_label: "Swiss German (all dialect regions)" locality: "per utterance: canton of dialect origin" # README.txt `accent`, one canton per utterance, carried in the sample_id metric: bleu+wer acquisition: Dropbox zip by URL, sha256-verified; durability vendor — a durable mirror is outstanding # ── Dialect probe + its control spur — read the publication rules below ── - id: xsid-bar # xSID-audio v0.2, Bavarian (rural Upper Bavaria) read speech zenodo: "21605015" doi: 10.5281/zenodo.21605015 version: "0.2" license: CC-BY-SA-4.0 (+ authors' no-speech-synthesis condition — see /NOTICE) language_tag: bar variety_label: "Bavarian" locality: "rural Upper Bavaria" # xSID-audio README; xSID's own label de-ba is not BCP 47 representativeness: low eligible_for_aggregate: false - id: xsid-de-control # SAME speaker, SAME sentences, Standard German zenodo: "21605015" doi: 10.5281/zenodo.21605015 version: "0.2" license: CC-BY-SA-4.0 (+ authors' no-speech-synthesis condition — see /NOTICE) language_tag: de variety_label: "Standard German (read by the xsid-bar speaker)" locality: null # the variety is Standard German, not local; the speaker's origin is xsid-bar's representativeness: low eligible_for_aggregate: false # How the dialect rows above may and may not be published. These are not style # preferences; each one names a way the number would otherwise mislead. dialect_publication_rules: # One speaker, read speech. The number answers "does the model collapse on # Bavarian at all", not "how good is it on Bavarian". - no dialect row ever carries a winner mark, however many rows exist. # xsid-bar alone measures one voice. xsid-bar minus xsid-de-control measures # the dialect. Ship the pair or ship neither. - a dialect row is published only alongside its same-speaker control row. # A mean over Swiss German and Bavarian describes no population. - never form an aggregate that spans two dialect areas. # eligible_for_aggregate=false in raven_asr.config carries this in code. - dialect datasets stay out of any cross-dataset or `overall` average. # A region row is a slice of one corpus, not a corpus: too few speakers to # rank anything, and a mean of region rows is a different estimator from the # corpus number (which pools utterances), weighted by nothing that exists. - a per-region row never carries a winner mark and is never averaged. # Per-locality rows INSIDE a dialect corpus. "Swiss German" is not one # evaluation category: a corpus that spans dialect regions is also reported per # region, next to — never instead of — its corpus number. Read out of committed # artifacts by `make analyse` (raven_asr.analysis.by_region), no re-run needed. dialect_region_breakdown: datasets: [fhnw-all-dialects] # every utterance carries its region; spc-test is one region (Bern), xSID one speaker region: canton of dialect origin # FHNW README.txt `accent`; BS speakers are labelled BL. Names per ISO 3166-2:CH source_of_region: sample_id # fhnw-all-dialects--, parsed by raven_asr.datasets.fhnw_all_dialects.region_of # Below this many utterances a region shows its n and a note, not a number. # 30 is a floor, not a precision target: the whole 50-utterance FHNW sample # already carries a ±7 pp WER interval, and a percentile bootstrap over a # handful of utterances mostly reports which few clips it drew. min_n: 30 wer: corpus WER over the region's utterances + percentile bootstrap CI (wer.uncertainty settings) bleu: corpus BLEU over the region's utterances + n — NO interval, BLEU has none yet coverage: every utterance lands in exactly one region; Σ region n == corpus n, or the breakdown refuses to print