{ "model": "openai-gpt-4o-mini-transcribe", "medical_canonicalization": "Canonical M-WER v2: reference typo fixes + reviewed low-risk morphology, with dangerous clinical/drug confusables kept penalized", "files": 55, "avg_wer": 0.136008, "avg_m_wer": 0.05743, "global_m_wer": 0.048571, "global_m_cer": 0.5972, "avg_medical_entity_recall": 0.9549, "global_medical_token_recall": 0.9514, "global_high_risk_m_wer": 0.096567, "global_high_risk_ref_count": 466, "global_per_category": { "drugs": { "ref_count": 466, "correct": 421, "substitutions": 31, "deletions": 14, "m_wer": 0.096567 }, "conditions": { "ref_count": 391, "correct": 372, "substitutions": 7, "deletions": 12, "m_wer": 0.048593 }, "symptoms": { "ref_count": 1162, "correct": 1119, "substitutions": 16, "deletions": 27, "m_wer": 0.037005 }, "anatomy": { "ref_count": 616, "correct": 597, "substitutions": 5, "deletions": 14, "m_wer": 0.030844 }, "clinical": { "ref_count": 493, "correct": 467, "substitutions": 4, "deletions": 22, "m_wer": 0.052738 } }, "total_medical_ref_words": 3150, "total_medical_correct": 2997, "total_medical_substitutions": 63, "total_medical_deletions": 90, "top_medical_errors": [ { "ref": "clenil", "hyp": "clenal", "count": 5 }, { "ref": "clenil", "hyp": "clonal", "count": 4 }, { "ref": "modulite", "hyp": "modulate", "count": 3 }, { "ref": "clenil", "hyp": "clenol", "count": 3 }, { "ref": "hyperthyroidism", "hyp": "hypothyroidism", "count": 2 }, { "ref": "implanon", "hyp": "on", "count": 2 }, { "ref": "infection", "hyp": "it", "count": 1 }, { "ref": "piriton", "hyp": "peritin", "count": 1 }, { "ref": "eczema", "hyp": "i", "count": 1 }, { "ref": "antihistamine", "hyp": "the", "count": 1 }, { "ref": "ibuprofen", "hyp": "and", "count": 1 }, { "ref": "tummy", "hyp": "primary", "count": 1 }, { "ref": "dose", "hyp": "day", "count": 1 }, { "ref": "bisoprolol", "hyp": "that", "count": 1 }, { "ref": "skin", "hyp": "the", "count": 1 }, { "ref": "piriton", "hyp": "piratin", "count": 1 }, { "ref": "loratadine", "hyp": "loratidine", "count": 1 }, { "ref": "urine", "hyp": "have", "count": 1 }, { "ref": "symptom", "hyp": "no", "count": 1 }, { "ref": "bowel", "hyp": "that", "count": 1 } ], "per_file": [ { "wer": 0.157049, "m_wer": 0.051282, "m_cer": 0.7778, "medical_entity_recall": 1.0, "medical_token_recall": 0.9615, "total_ref_words": 1369, "total_medical_ref_words": 78, "medical_correct": 75, "medical_substitutions": 1, "medical_deletions": 2, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "vomit", "file": "day1_consultation01" }, { "type": "deletion", "reference": "vomit", "category": "symptoms", "file": "day1_consultation01" }, { "type": "deletion", "reference": "fever", "category": "symptoms", "file": "day1_consultation01" }, { "type": "substitution", "reference": "infection", "hypothesis": "it", "category": "conditions", "cer": 0.7778, "file": "day1_consultation01" } ], "unique_medical_terms_in_ref": 25, "unique_medical_terms_found": 25, "per_category": { "drugs": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 11, "correct": 10, "substitutions": 1, "deletions": 0, "m_wer": 0.090909 }, "symptoms": { "ref_count": 39, "correct": 37, "substitutions": 0, "deletions": 2, "m_wer": 0.051282 }, "anatomy": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 8, "file": "day1_consultation01" }, { "wer": 0.142174, "m_wer": 0.043011, "m_cer": 0.685, "medical_entity_recall": 0.9667, "medical_token_recall": 0.957, "total_ref_words": 1674, "total_medical_ref_words": 93, "medical_correct": 89, "medical_substitutions": 3, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "itchy", "category": "symptoms", "file": "day1_consultation02" }, { "type": "substitution", "reference": "piriton", "hypothesis": "peritin", "category": "drugs", "cer": 0.2857, "file": "day1_consultation02" }, { "type": "substitution", "reference": "eczema", "hypothesis": "i", "category": "conditions", "cer": 1.0, "file": "day1_consultation02" }, { "type": "substitution", "reference": "antihistamine", "hypothesis": "the", "category": "drugs", "cer": 0.7692, "file": "day1_consultation02" } ], "unique_medical_terms_in_ref": 30, "unique_medical_terms_found": 29, "per_category": { "drugs": { "ref_count": 15, "correct": 13, "substitutions": 2, "deletions": 0, "m_wer": 0.133333 }, "conditions": { "ref_count": 10, "correct": 9, "substitutions": 1, "deletions": 0, "m_wer": 0.1 }, "symptoms": { "ref_count": 36, "correct": 35, "substitutions": 0, "deletions": 1, "m_wer": 0.027778 }, "anatomy": { "ref_count": 21, "correct": 21, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.133333, "high_risk_ref_count": 15, "file": "day1_consultation02" }, { "wer": 0.217891, "m_wer": 0.083333, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9306, "total_ref_words": 1565, "total_medical_ref_words": 72, "medical_correct": 67, "medical_substitutions": 0, "medical_deletions": 5, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "deletion", "reference": "pain", "category": "symptoms", "file": "day1_consultation03" }, { "type": "deletion", "reference": "headache", "category": "symptoms", "file": "day1_consultation03" }, { "type": "deletion", "reference": "pain", "category": "symptoms", "file": "day1_consultation03" }, { "type": "deletion", "reference": "headache", "category": "symptoms", "file": "day1_consultation03" }, { "type": "deletion", "reference": "cancer", "category": "conditions", "file": "day1_consultation03" }, { "type": "insertion", "hypothesis": "migraine", "file": "day1_consultation03" } ], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 26, "per_category": { "drugs": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 12, "correct": 11, "substitutions": 0, "deletions": 1, "m_wer": 0.083333 }, "symptoms": { "ref_count": 38, "correct": 34, "substitutions": 0, "deletions": 4, "m_wer": 0.105263 }, "anatomy": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 8, "file": "day1_consultation03" }, { "wer": 0.177253, "m_wer": 0.014493, "m_cer": 1.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9855, "total_ref_words": 1653, "total_medical_ref_words": 69, "medical_correct": 68, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "ibuprofen", "hypothesis": "and", "category": "drugs", "cer": 1.0, "file": "day1_consultation04" } ], "unique_medical_terms_in_ref": 32, "unique_medical_terms_found": 32, "per_category": { "drugs": { "ref_count": 5, "correct": 4, "substitutions": 1, "deletions": 0, "m_wer": 0.2 }, "conditions": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 32, "correct": 32, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 19, "correct": 19, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.2, "high_risk_ref_count": 5, "file": "day1_consultation04" }, { "wer": 0.146789, "m_wer": 0.011494, "m_cer": 1.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9885, "total_ref_words": 1635, "total_medical_ref_words": 87, "medical_correct": 86, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "tummy", "hypothesis": "primary", "category": "anatomy", "cer": 1.0, "file": "day1_consultation05" } ], "unique_medical_terms_in_ref": 31, "unique_medical_terms_found": 31, "per_category": { "drugs": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 32, "correct": 32, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 24, "correct": 23, "substitutions": 1, "deletions": 0, "m_wer": 0.041667 }, "clinical": { "ref_count": 13, "correct": 13, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 14, "file": "day1_consultation05" }, { "wer": 0.152422, "m_wer": 0.076923, "m_cer": 0.875, "medical_entity_recall": 0.9355, "medical_token_recall": 0.9231, "total_ref_words": 1837, "total_medical_ref_words": 91, "medical_correct": 84, "medical_substitutions": 2, "medical_deletions": 5, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "medicine", "category": "drugs", "file": "day1_consultation06" }, { "type": "deletion", "reference": "cough", "category": "symptoms", "file": "day1_consultation06" }, { "type": "deletion", "reference": "ankles", "category": "anatomy", "file": "day1_consultation06" }, { "type": "deletion", "reference": "heart", "category": "anatomy", "file": "day1_consultation06" }, { "type": "substitution", "reference": "dose", "hypothesis": "day", "category": "clinical", "cer": 0.75, "file": "day1_consultation06" }, { "type": "deletion", "reference": "bisoprolol", "category": "drugs", "file": "day1_consultation06" }, { "type": "substitution", "reference": "bisoprolol", "hypothesis": "that", "category": "drugs", "cer": 1.0, "file": "day1_consultation06" } ], "unique_medical_terms_in_ref": 31, "unique_medical_terms_found": 29, "per_category": { "drugs": { "ref_count": 10, "correct": 7, "substitutions": 1, "deletions": 2, "m_wer": 0.3 }, "conditions": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 25, "correct": 24, "substitutions": 0, "deletions": 1, "m_wer": 0.04 }, "anatomy": { "ref_count": 38, "correct": 36, "substitutions": 0, "deletions": 2, "m_wer": 0.052632 }, "clinical": { "ref_count": 16, "correct": 15, "substitutions": 1, "deletions": 0, "m_wer": 0.0625 } }, "high_risk_m_wer": 0.3, "high_risk_ref_count": 10, "file": "day1_consultation06" }, { "wer": 0.128977, "m_wer": 0.065574, "m_cer": 0.4619, "medical_entity_recall": 0.931, "medical_token_recall": 0.9344, "total_ref_words": 1163, "total_medical_ref_words": 61, "medical_correct": 57, "medical_substitutions": 3, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "medical", "category": "clinical", "file": "day1_consultation08" }, { "type": "substitution", "reference": "skin", "hypothesis": "the", "category": "anatomy", "cer": 1.0, "file": "day1_consultation08" }, { "type": "substitution", "reference": "piriton", "hypothesis": "piratin", "category": "drugs", "cer": 0.2857, "file": "day1_consultation08" }, { "type": "substitution", "reference": "loratadine", "hypothesis": "loratidine", "category": "drugs", "cer": 0.1, "file": "day1_consultation08" } ], "unique_medical_terms_in_ref": 29, "unique_medical_terms_found": 27, "per_category": { "drugs": { "ref_count": 11, "correct": 9, "substitutions": 2, "deletions": 0, "m_wer": 0.181818 }, "conditions": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 22, "correct": 22, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 15, "correct": 14, "substitutions": 1, "deletions": 0, "m_wer": 0.066667 }, "clinical": { "ref_count": 8, "correct": 7, "substitutions": 0, "deletions": 1, "m_wer": 0.125 } }, "high_risk_m_wer": 0.181818, "high_risk_ref_count": 11, "file": "day1_consultation08" }, { "wer": 0.176735, "m_wer": 0.014085, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9859, "total_ref_words": 1556, "total_medical_ref_words": 71, "medical_correct": 70, "medical_substitutions": 0, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "medical", "category": "clinical", "file": "day1_consultation09" } ], "unique_medical_terms_in_ref": 27, "unique_medical_terms_found": 27, "per_category": { "drugs": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 34, "correct": 34, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 17, "correct": 17, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 9, "correct": 8, "substitutions": 0, "deletions": 1, "m_wer": 0.111111 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 6, "file": "day1_consultation09" }, { "wer": 0.165409, "m_wer": 0.066667, "m_cer": 0.8857, "medical_entity_recall": 0.96, "medical_token_recall": 0.9467, "total_ref_words": 1856, "total_medical_ref_words": 75, "medical_correct": 71, "medical_substitutions": 3, "medical_deletions": 1, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "urine", "file": "day1_consultation10" }, { "type": "substitution", "reference": "urine", "hypothesis": "have", "category": "anatomy", "cer": 0.8, "file": "day1_consultation10" }, { "type": "substitution", "reference": "symptom", "hypothesis": "no", "category": "symptoms", "cer": 0.8571, "file": "day1_consultation10" }, { "type": "substitution", "reference": "bowel", "hypothesis": "that", "category": "anatomy", "cer": 1.0, "file": "day1_consultation10" }, { "type": "deletion", "reference": "cramps", "category": "symptoms", "file": "day1_consultation10" } ], "unique_medical_terms_in_ref": 25, "unique_medical_terms_found": 24, "per_category": { "drugs": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 37, "correct": 35, "substitutions": 1, "deletions": 1, "m_wer": 0.054054 }, "anatomy": { "ref_count": 24, "correct": 22, "substitutions": 2, "deletions": 0, "m_wer": 0.083333 }, "clinical": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 9, "file": "day1_consultation10" }, { "wer": 0.300086, "m_wer": 0.28, "m_cer": 0.5, "medical_entity_recall": 0.7407, "medical_token_recall": 0.72, "total_ref_words": 2336, "total_medical_ref_words": 75, "medical_correct": 54, "medical_substitutions": 1, "medical_deletions": 20, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "crampy", "hypothesis": "cramping", "category": "symptoms", "cer": 0.5, "file": "day1_consultation11" }, { "type": "deletion", "reference": "bowel", "category": "anatomy", "file": "day1_consultation11" }, { "type": "deletion", "reference": "symptom", "category": "symptoms", "file": "day1_consultation11" }, { "type": "deletion", "reference": "gastroenteritis", "category": "conditions", "file": "day1_consultation11" }, { "type": "deletion", "reference": "infection", "category": "conditions", "file": "day1_consultation11" }, { "type": "deletion", "reference": "tummy", "category": "anatomy", "file": "day1_consultation11" }, { "type": "deletion", "reference": "fluid", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "fluid", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "fluid", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "pharmacy", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "prescription", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "paracetamol", "category": "drugs", "file": "day1_consultation11" }, { "type": "deletion", "reference": "feverish", "category": "symptoms", "file": "day1_consultation11" }, { "type": "deletion", "reference": "symptom", "category": "symptoms", "file": "day1_consultation11" }, { "type": "deletion", "reference": "fluid", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "fluid", "category": "clinical", "file": "day1_consultation11" }, { "type": "deletion", "reference": "paracetamol", "category": "drugs", "file": "day1_consultation11" }, { "type": "deletion", "reference": "tummy", "category": "anatomy", "file": "day1_consultation11" }, { "type": "deletion", "reference": "pain", "category": "symptoms", "file": "day1_consultation11" }, { "type": "deletion", "reference": "feverish", "category": "symptoms", "file": "day1_consultation11" }, { "type": "deletion", "reference": "paracetamol", "category": "drugs", "file": "day1_consultation11" } ], "unique_medical_terms_in_ref": 27, "unique_medical_terms_found": 20, "per_category": { "drugs": { "ref_count": 6, "correct": 3, "substitutions": 0, "deletions": 3, "m_wer": 0.5 }, "conditions": { "ref_count": 10, "correct": 8, "substitutions": 0, "deletions": 2, "m_wer": 0.2 }, "symptoms": { "ref_count": 30, "correct": 24, "substitutions": 1, "deletions": 5, "m_wer": 0.2 }, "anatomy": { "ref_count": 13, "correct": 10, "substitutions": 0, "deletions": 3, "m_wer": 0.230769 }, "clinical": { "ref_count": 16, "correct": 9, "substitutions": 0, "deletions": 7, "m_wer": 0.4375 } }, "high_risk_m_wer": 0.5, "high_risk_ref_count": 6, "file": "day1_consultation11" }, { "wer": 0.125541, "m_wer": 0.019608, "m_cer": 0.5714, "medical_entity_recall": 1.0, "medical_token_recall": 0.9804, "total_ref_words": 1155, "total_medical_ref_words": 51, "medical_correct": 50, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "inhaler", "hypothesis": "nailers", "category": "drugs", "cer": 0.5714, "file": "day1_consultation12" } ], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 26, "per_category": { "drugs": { "ref_count": 8, "correct": 7, "substitutions": 1, "deletions": 0, "m_wer": 0.125 }, "conditions": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.125, "high_risk_ref_count": 8, "file": "day1_consultation12" }, { "wer": 0.101896, "m_wer": 0.015152, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1688, "total_medical_ref_words": 66, "medical_correct": 66, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "codeine", "file": "day1_consultation13" } ], "unique_medical_terms_in_ref": 24, "unique_medical_terms_found": 24, "per_category": { "drugs": { "ref_count": 15, "correct": 15, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 34, "correct": 34, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 15, "file": "day1_consultation13" }, { "wer": 0.167179, "m_wer": 0.02439, "m_cer": 0.75, "medical_entity_recall": 1.0, "medical_token_recall": 0.9756, "total_ref_words": 1950, "total_medical_ref_words": 82, "medical_correct": 80, "medical_substitutions": 1, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "discharge", "category": "symptoms", "file": "day1_consultation14" }, { "type": "substitution", "reference": "diabetes", "hypothesis": "yes", "category": "conditions", "cer": 0.75, "file": "day1_consultation14" } ], "unique_medical_terms_in_ref": 34, "unique_medical_terms_found": 34, "per_category": { "drugs": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 11, "correct": 10, "substitutions": 1, "deletions": 0, "m_wer": 0.090909 }, "symptoms": { "ref_count": 34, "correct": 33, "substitutions": 0, "deletions": 1, "m_wer": 0.029412 }, "anatomy": { "ref_count": 20, "correct": 20, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 7, "file": "day1_consultation14" }, { "wer": 0.394488, "m_wer": 0.173913, "m_cer": 1.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9783, "total_ref_words": 1270, "total_medical_ref_words": 46, "medical_correct": 45, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 7, "spurious_medical_insertions": 7, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "symptom", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "itchy", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "itchy", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "bleeding", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "skin", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "symptom", "file": "day1_consultation15" }, { "type": "insertion", "hypothesis": "metformin", "file": "day1_consultation15" }, { "type": "substitution", "reference": "antihistamine", "hypothesis": "ok", "category": "drugs", "cer": 1.0, "file": "day1_consultation15" } ], "unique_medical_terms_in_ref": 25, "unique_medical_terms_found": 25, "per_category": { "drugs": { "ref_count": 8, "correct": 7, "substitutions": 1, "deletions": 0, "m_wer": 0.125 }, "conditions": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 15, "correct": 15, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.125, "high_risk_ref_count": 8, "file": "day1_consultation15" }, { "wer": 0.101709, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1170, "total_medical_ref_words": 15, "medical_correct": 15, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 12, "unique_medical_terms_found": 12, "per_category": { "drugs": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 0, "correct": 0, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 1, "file": "day2_consultation01" }, { "wer": 0.131517, "m_wer": 0.057692, "m_cer": 0.0, "medical_entity_recall": 0.9524, "medical_token_recall": 0.9423, "total_ref_words": 1688, "total_medical_ref_words": 52, "medical_correct": 49, "medical_substitutions": 0, "medical_deletions": 3, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "medical", "category": "clinical", "file": "day2_consultation02" }, { "type": "deletion", "reference": "skin", "category": "anatomy", "file": "day2_consultation02" }, { "type": "deletion", "reference": "blood", "category": "anatomy", "file": "day2_consultation02" } ], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 20, "per_category": { "drugs": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 19, "correct": 17, "substitutions": 0, "deletions": 2, "m_wer": 0.105263 }, "clinical": { "ref_count": 15, "correct": 14, "substitutions": 0, "deletions": 1, "m_wer": 0.066667 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 4, "file": "day2_consultation02" }, { "wer": 0.142556, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1424, "total_medical_ref_words": 19, "medical_correct": 19, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 11, "unique_medical_terms_found": 11, "per_category": { "drugs": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 0, "correct": 0, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 0, "correct": 0, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 3, "file": "day2_consultation03" }, { "wer": 0.108493, "m_wer": 0.029412, "m_cer": 0.8, "medical_entity_recall": 1.0, "medical_token_recall": 0.9706, "total_ref_words": 1825, "total_medical_ref_words": 68, "medical_correct": 66, "medical_substitutions": 1, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "cough", "hypothesis": "up", "category": "symptoms", "cer": 0.8, "file": "day2_consultation04" }, { "type": "deletion", "reference": "cancer", "category": "conditions", "file": "day2_consultation04" } ], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 21, "per_category": { "drugs": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 10, "correct": 9, "substitutions": 0, "deletions": 1, "m_wer": 0.1 }, "symptoms": { "ref_count": 16, "correct": 15, "substitutions": 1, "deletions": 0, "m_wer": 0.0625 }, "anatomy": { "ref_count": 29, "correct": 29, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 6, "file": "day2_consultation04" }, { "wer": 0.091288, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1917, "total_medical_ref_words": 34, "medical_correct": 34, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 18, "unique_medical_terms_found": 18, "per_category": { "drugs": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 5, "file": "day2_consultation05" }, { "wer": 0.118163, "m_wer": 0.03125, "m_cer": 0.4286, "medical_entity_recall": 0.9231, "medical_token_recall": 0.9688, "total_ref_words": 1938, "total_medical_ref_words": 64, "medical_correct": 62, "medical_substitutions": 1, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "piriton", "hypothesis": "pyridone", "category": "drugs", "cer": 0.4286, "file": "day2_consultation06" }, { "type": "deletion", "reference": "pharmacy", "category": "clinical", "file": "day2_consultation06" } ], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 24, "per_category": { "drugs": { "ref_count": 8, "correct": 7, "substitutions": 1, "deletions": 0, "m_wer": 0.125 }, "conditions": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 18, "correct": 18, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 26, "correct": 25, "substitutions": 0, "deletions": 1, "m_wer": 0.038462 } }, "high_risk_m_wer": 0.125, "high_risk_ref_count": 8, "file": "day2_consultation06" }, { "wer": 0.130137, "m_wer": 0.045455, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 0.9545, "total_ref_words": 1314, "total_medical_ref_words": 44, "medical_correct": 42, "medical_substitutions": 0, "medical_deletions": 2, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "blood", "category": "anatomy", "file": "day2_consultation07" }, { "type": "deletion", "reference": "allergy", "category": "clinical", "file": "day2_consultation07" } ], "unique_medical_terms_in_ref": 16, "unique_medical_terms_found": 16, "per_category": { "drugs": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 19, "correct": 18, "substitutions": 0, "deletions": 1, "m_wer": 0.052632 }, "clinical": { "ref_count": 8, "correct": 7, "substitutions": 0, "deletions": 1, "m_wer": 0.125 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 4, "file": "day2_consultation07" }, { "wer": 0.095317, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1815, "total_medical_ref_words": 74, "medical_correct": 74, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 32, "unique_medical_terms_found": 32, "per_category": { "drugs": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 30, "correct": 30, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 9, "file": "day2_consultation08" }, { "wer": 0.123153, "m_wer": 0.076923, "m_cer": 0.1111, "medical_entity_recall": 0.8889, "medical_token_recall": 0.9231, "total_ref_words": 1015, "total_medical_ref_words": 13, "medical_correct": 12, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "metformin", "hypothesis": "melformin", "category": "drugs", "cer": 0.1111, "file": "day2_consultation09" } ], "unique_medical_terms_in_ref": 9, "unique_medical_terms_found": 8, "per_category": { "drugs": { "ref_count": 3, "correct": 2, "substitutions": 1, "deletions": 0, "m_wer": 0.333333 }, "conditions": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.333333, "high_risk_ref_count": 3, "file": "day2_consultation09" }, { "wer": 0.087661, "m_wer": 0.033333, "m_cer": 0.875, "medical_entity_recall": 1.0, "medical_token_recall": 0.9833, "total_ref_words": 1791, "total_medical_ref_words": 60, "medical_correct": 59, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "headache", "file": "day2_consultation10" }, { "type": "substitution", "reference": "headache", "hypothesis": "that", "category": "symptoms", "cer": 0.875, "file": "day2_consultation10" } ], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 21, "per_category": { "drugs": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 22, "correct": 21, "substitutions": 1, "deletions": 0, "m_wer": 0.045455 }, "anatomy": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 14, "file": "day2_consultation10" }, { "wer": 0.143975, "m_wer": 0.021739, "m_cer": 0.1429, "medical_entity_recall": 0.9412, "medical_token_recall": 0.9783, "total_ref_words": 1278, "total_medical_ref_words": 46, "medical_correct": 45, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "piriton", "hypothesis": "periton", "category": "drugs", "cer": 0.1429, "file": "day3_consultation01" } ], "unique_medical_terms_in_ref": 17, "unique_medical_terms_found": 16, "per_category": { "drugs": { "ref_count": 14, "correct": 13, "substitutions": 1, "deletions": 0, "m_wer": 0.071429 }, "conditions": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 12, "correct": 12, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.071429, "high_risk_ref_count": 14, "file": "day3_consultation01" }, { "wer": 0.12807, "m_wer": 0.020833, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1140, "total_medical_ref_words": 48, "medical_correct": 48, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "ibuprofen", "file": "day3_consultation02" } ], "unique_medical_terms_in_ref": 18, "unique_medical_terms_found": 18, "per_category": { "drugs": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 0, "correct": 0, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 34, "correct": 34, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 4, "file": "day3_consultation02" }, { "wer": 0.123989, "m_wer": 0.175439, "m_cer": 0.4333, "medical_entity_recall": 0.8696, "medical_token_recall": 0.8947, "total_ref_words": 1113, "total_medical_ref_words": 57, "medical_correct": 51, "medical_substitutions": 6, "medical_deletions": 0, "medical_insertions": 4, "spurious_medical_insertions": 4, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "salbutamol", "file": "day3_consultation04" }, { "type": "substitution", "reference": "wheezy", "hypothesis": "wheezing", "category": "symptoms", "cer": 0.5, "file": "day3_consultation04" }, { "type": "insertion", "hypothesis": "clenil", "file": "day3_consultation04" }, { "type": "insertion", "hypothesis": "asthma", "file": "day3_consultation04" }, { "type": "substitution", "reference": "fever", "hypothesis": "hayfever", "category": "symptoms", "cer": 0.6, "file": "day3_consultation04" }, { "type": "insertion", "hypothesis": "inhaler", "file": "day3_consultation04" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clonal", "category": "drugs", "cer": 0.3333, "file": "day3_consultation04" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clonal", "category": "drugs", "cer": 0.3333, "file": "day3_consultation04" }, { "type": "substitution", "reference": "sneeze", "hypothesis": "sees", "category": "symptoms", "cer": 0.5, "file": "day3_consultation04" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clonal", "category": "drugs", "cer": 0.3333, "file": "day3_consultation04" } ], "unique_medical_terms_in_ref": 23, "unique_medical_terms_found": 20, "per_category": { "drugs": { "ref_count": 22, "correct": 19, "substitutions": 3, "deletions": 0, "m_wer": 0.136364 }, "conditions": { "ref_count": 12, "correct": 12, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 11, "correct": 8, "substitutions": 3, "deletions": 0, "m_wer": 0.272727 }, "anatomy": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.136364, "high_risk_ref_count": 22, "file": "day3_consultation04" }, { "wer": 0.103295, "m_wer": 0.074074, "m_cer": 0.8333, "medical_entity_recall": 1.0, "medical_token_recall": 0.963, "total_ref_words": 1123, "total_medical_ref_words": 27, "medical_correct": 26, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "cough", "file": "day3_consultation05" }, { "type": "substitution", "reference": "prescription", "hypothesis": "across", "category": "clinical", "cer": 0.8333, "file": "day3_consultation05" } ], "unique_medical_terms_in_ref": 17, "unique_medical_terms_found": 17, "per_category": { "drugs": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 8, "correct": 7, "substitutions": 1, "deletions": 0, "m_wer": 0.125 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 6, "file": "day3_consultation05" }, { "wer": 0.118928, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 597, "total_medical_ref_words": 24, "medical_correct": 24, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 15, "unique_medical_terms_found": 15, "per_category": { "drugs": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 3, "file": "day3_consultation06" }, { "wer": 0.144915, "m_wer": 0.067797, "m_cer": 0.5667, "medical_entity_recall": 0.92, "medical_token_recall": 0.9661, "total_ref_words": 1180, "total_medical_ref_words": 59, "medical_correct": 57, "medical_substitutions": 2, "medical_deletions": 0, "medical_insertions": 2, "spurious_medical_insertions": 2, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "eczema", "file": "day3_consultation07" }, { "type": "insertion", "hypothesis": "joint", "file": "day3_consultation07" }, { "type": "substitution", "reference": "fever", "hypothesis": "pain", "category": "symptoms", "cer": 1.0, "file": "day3_consultation07" }, { "type": "substitution", "reference": "hyperthyroidism", "hypothesis": "hypothyroidism", "category": "conditions", "cer": 0.1333, "file": "day3_consultation07" } ], "unique_medical_terms_in_ref": 25, "unique_medical_terms_found": 23, "per_category": { "drugs": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 15, "correct": 14, "substitutions": 1, "deletions": 0, "m_wer": 0.066667 }, "symptoms": { "ref_count": 11, "correct": 10, "substitutions": 1, "deletions": 0, "m_wer": 0.090909 }, "anatomy": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 16, "file": "day3_consultation07" }, { "wer": 0.137106, "m_wer": 0.04, "m_cer": 0.9375, "medical_entity_recall": 0.9583, "medical_token_recall": 0.96, "total_ref_words": 919, "total_medical_ref_words": 50, "medical_correct": 48, "medical_substitutions": 2, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "headache", "hypothesis": "like", "category": "symptoms", "cer": 0.875, "file": "day3_consultation08" }, { "type": "substitution", "reference": "cough", "hypothesis": "is", "category": "symptoms", "cer": 1.0, "file": "day3_consultation08" } ], "unique_medical_terms_in_ref": 24, "unique_medical_terms_found": 23, "per_category": { "drugs": { "ref_count": 12, "correct": 12, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 15, "correct": 13, "substitutions": 2, "deletions": 0, "m_wer": 0.133333 }, "anatomy": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 12, "file": "day3_consultation08" }, { "wer": 0.197843, "m_wer": 0.162791, "m_cer": 0.0, "medical_entity_recall": 0.7619, "medical_token_recall": 0.8372, "total_ref_words": 2133, "total_medical_ref_words": 43, "medical_correct": 36, "medical_substitutions": 0, "medical_deletions": 7, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "drug", "category": "clinical", "file": "day3_consultation09" }, { "type": "deletion", "reference": "steroid", "category": "drugs", "file": "day3_consultation09" }, { "type": "deletion", "reference": "acute", "category": "clinical", "file": "day3_consultation09" }, { "type": "deletion", "reference": "acute", "category": "clinical", "file": "day3_consultation09" }, { "type": "deletion", "reference": "headache", "category": "symptoms", "file": "day3_consultation09" }, { "type": "deletion", "reference": "vomiting", "category": "symptoms", "file": "day3_consultation09" }, { "type": "deletion", "reference": "symptom", "category": "symptoms", "file": "day3_consultation09" } ], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 16, "per_category": { "drugs": { "ref_count": 6, "correct": 5, "substitutions": 0, "deletions": 1, "m_wer": 0.166667 }, "conditions": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 26, "correct": 23, "substitutions": 0, "deletions": 3, "m_wer": 0.115385 }, "anatomy": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 2, "substitutions": 0, "deletions": 3, "m_wer": 0.6 } }, "high_risk_m_wer": 0.166667, "high_risk_ref_count": 6, "file": "day3_consultation09" }, { "wer": 0.069322, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1356, "total_medical_ref_words": 36, "medical_correct": 36, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 16, "unique_medical_terms_found": 16, "per_category": { "drugs": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 18, "correct": 18, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 4, "file": "day3_consultation10" }, { "wer": 0.127159, "m_wer": 0.096154, "m_cer": 0.5, "medical_entity_recall": 0.9444, "medical_token_recall": 0.9231, "total_ref_words": 1274, "total_medical_ref_words": 52, "medical_correct": 48, "medical_substitutions": 1, "medical_deletions": 3, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "deletion", "reference": "pain", "category": "symptoms", "file": "day4_consultation01" }, { "type": "deletion", "reference": "hypertension", "category": "conditions", "file": "day4_consultation01" }, { "type": "substitution", "reference": "medicine", "hypothesis": "medication", "category": "drugs", "cer": 0.5, "file": "day4_consultation01" }, { "type": "deletion", "reference": "medicine", "category": "drugs", "file": "day4_consultation01" }, { "type": "insertion", "hypothesis": "asthma", "file": "day4_consultation01" } ], "unique_medical_terms_in_ref": 18, "unique_medical_terms_found": 17, "per_category": { "drugs": { "ref_count": 3, "correct": 1, "substitutions": 1, "deletions": 1, "m_wer": 0.666667 }, "conditions": { "ref_count": 5, "correct": 4, "substitutions": 0, "deletions": 1, "m_wer": 0.2 }, "symptoms": { "ref_count": 14, "correct": 13, "substitutions": 0, "deletions": 1, "m_wer": 0.071429 }, "anatomy": { "ref_count": 20, "correct": 20, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.666667, "high_risk_ref_count": 3, "file": "day4_consultation01" }, { "wer": 0.090836, "m_wer": 0.035714, "m_cer": 0.1333, "medical_entity_recall": 0.9231, "medical_token_recall": 0.9643, "total_ref_words": 1244, "total_medical_ref_words": 28, "medical_correct": 27, "medical_substitutions": 1, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "hyperthyroidism", "hypothesis": "hypothyroidism", "category": "conditions", "cer": 0.1333, "file": "day4_consultation02" } ], "unique_medical_terms_in_ref": 13, "unique_medical_terms_found": 12, "per_category": { "drugs": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 1, "correct": 0, "substitutions": 1, "deletions": 0, "m_wer": 1.0 }, "symptoms": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 5, "file": "day4_consultation02" }, { "wer": 0.059629, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1023, "total_medical_ref_words": 37, "medical_correct": 37, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 15, "unique_medical_terms_found": 15, "per_category": { "drugs": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 0, "correct": 0, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 12, "correct": 12, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 7, "file": "day4_consultation03" }, { "wer": 0.081054, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 987, "total_medical_ref_words": 42, "medical_correct": 42, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 19, "unique_medical_terms_found": 19, "per_category": { "drugs": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 7, "file": "day4_consultation04" }, { "wer": 0.192455, "m_wer": 0.027027, "m_cer": 0.0, "medical_entity_recall": 0.9583, "medical_token_recall": 0.973, "total_ref_words": 1564, "total_medical_ref_words": 74, "medical_correct": 72, "medical_substitutions": 0, "medical_deletions": 2, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "inhaler", "category": "drugs", "file": "day4_consultation05" }, { "type": "deletion", "reference": "bowel", "category": "anatomy", "file": "day4_consultation05" } ], "unique_medical_terms_in_ref": 24, "unique_medical_terms_found": 23, "per_category": { "drugs": { "ref_count": 8, "correct": 7, "substitutions": 0, "deletions": 1, "m_wer": 0.125 }, "conditions": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 25, "correct": 25, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 23, "correct": 22, "substitutions": 0, "deletions": 1, "m_wer": 0.043478 }, "clinical": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.125, "high_risk_ref_count": 8, "file": "day4_consultation05" }, { "wer": 0.104242, "m_wer": 0.013514, "m_cer": 0.0, "medical_entity_recall": 0.9524, "medical_token_recall": 0.9865, "total_ref_words": 1650, "total_medical_ref_words": 74, "medical_correct": 73, "medical_substitutions": 0, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "hyperthyroid", "category": "conditions", "file": "day4_consultation06" } ], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 20, "per_category": { "drugs": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 19, "correct": 18, "substitutions": 0, "deletions": 1, "m_wer": 0.052632 }, "symptoms": { "ref_count": 30, "correct": 30, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 16, "file": "day4_consultation06" }, { "wer": 0.131232, "m_wer": 0.01087, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1745, "total_medical_ref_words": 92, "medical_correct": 92, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "medical", "file": "day4_consultation07" } ], "unique_medical_terms_in_ref": 25, "unique_medical_terms_found": 25, "per_category": { "drugs": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 38, "correct": 38, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 22, "correct": 22, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 20, "correct": 20, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 4, "file": "day4_consultation07" }, { "wer": 0.128099, "m_wer": 0.064516, "m_cer": 0.494, "medical_entity_recall": 0.9375, "medical_token_recall": 0.9355, "total_ref_words": 1694, "total_medical_ref_words": 31, "medical_correct": 29, "medical_substitutions": 2, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "hyperthyroid", "hypothesis": "hypothyroidism", "category": "conditions", "cer": 0.4167, "file": "day4_consultation08" }, { "type": "substitution", "reference": "thyroid", "hypothesis": "hypothyroid", "category": "anatomy", "cer": 0.5714, "file": "day4_consultation08" } ], "unique_medical_terms_in_ref": 16, "unique_medical_terms_found": 15, "per_category": { "drugs": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 5, "correct": 4, "substitutions": 1, "deletions": 0, "m_wer": 0.2 }, "symptoms": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 2, "correct": 1, "substitutions": 1, "deletions": 0, "m_wer": 0.5 }, "clinical": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 6, "file": "day4_consultation08" }, { "wer": 0.117786, "m_wer": 0.134454, "m_cer": 0.234, "medical_entity_recall": 0.9412, "medical_token_recall": 0.8655, "total_ref_words": 1698, "total_medical_ref_words": 119, "medical_correct": 103, "medical_substitutions": 13, "medical_deletions": 3, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "cough", "hypothesis": "like", "category": "symptoms", "cer": 1.0, "file": "day4_consultation09" }, { "type": "deletion", "reference": "cough", "category": "symptoms", "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenal", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenal", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "deletion", "reference": "clenil", "category": "drugs", "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenal", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "modulite", "hypothesis": "modulate", "category": "drugs", "cer": 0.125, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenal", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenal", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenol", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "modulite", "hypothesis": "modulate", "category": "drugs", "cer": 0.125, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenol", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "deletion", "reference": "steroid", "category": "drugs", "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clenol", "category": "drugs", "cer": 0.1667, "file": "day4_consultation09" }, { "type": "substitution", "reference": "clenil", "hypothesis": "clonal", "category": "drugs", "cer": 0.3333, "file": "day4_consultation09" }, { "type": "substitution", "reference": "modulite", "hypothesis": "modulate", "category": "drugs", "cer": 0.125, "file": "day4_consultation09" } ], "unique_medical_terms_in_ref": 34, "unique_medical_terms_found": 32, "per_category": { "drugs": { "ref_count": 41, "correct": 27, "substitutions": 12, "deletions": 2, "m_wer": 0.341463 }, "conditions": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 25, "correct": 23, "substitutions": 1, "deletions": 1, "m_wer": 0.08 }, "anatomy": { "ref_count": 12, "correct": 12, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 26, "correct": 26, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.341463, "high_risk_ref_count": 41, "file": "day4_consultation09" }, { "wer": 0.111444, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1669, "total_medical_ref_words": 74, "medical_correct": 74, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 26, "per_category": { "drugs": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 35, "correct": 35, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 7, "file": "day4_consultation10" }, { "wer": 0.097424, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1242, "total_medical_ref_words": 53, "medical_correct": 53, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 28, "unique_medical_terms_found": 28, "per_category": { "drugs": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 26, "correct": 26, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 11, "file": "day5_consultation01" }, { "wer": 0.070111, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1084, "total_medical_ref_words": 57, "medical_correct": 57, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 21, "unique_medical_terms_found": 21, "per_category": { "drugs": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 6, "file": "day5_consultation02" }, { "wer": 0.250804, "m_wer": 0.515152, "m_cer": 1.0, "medical_entity_recall": 0.5385, "medical_token_recall": 0.4848, "total_ref_words": 2488, "total_medical_ref_words": 33, "medical_correct": 16, "medical_substitutions": 1, "medical_deletions": 16, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "heart", "category": "anatomy", "file": "day5_consultation03" }, { "type": "substitution", "reference": "drug", "hypothesis": "them", "category": "clinical", "cer": 1.0, "file": "day5_consultation03" }, { "type": "deletion", "reference": "depression", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "depression", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "therapy", "category": "clinical", "file": "day5_consultation03" }, { "type": "deletion", "reference": "therapy", "category": "clinical", "file": "day5_consultation03" }, { "type": "deletion", "reference": "anxiety", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "straightaway", "category": null, "file": "day5_consultation03" }, { "type": "deletion", "reference": "anxiety", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "anxiety", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "medication", "category": "drugs", "file": "day5_consultation03" }, { "type": "deletion", "reference": "anxiety", "category": "conditions", "file": "day5_consultation03" }, { "type": "deletion", "reference": "blood", "category": "anatomy", "file": "day5_consultation03" }, { "type": "deletion", "reference": "test", "category": "clinical", "file": "day5_consultation03" }, { "type": "deletion", "reference": "test", "category": "clinical", "file": "day5_consultation03" }, { "type": "deletion", "reference": "blood", "category": "anatomy", "file": "day5_consultation03" }, { "type": "deletion", "reference": "test", "category": "clinical", "file": "day5_consultation03" } ], "unique_medical_terms_in_ref": 13, "unique_medical_terms_found": 7, "per_category": { "drugs": { "ref_count": 1, "correct": 0, "substitutions": 0, "deletions": 1, "m_wer": 1.0 }, "conditions": { "ref_count": 10, "correct": 4, "substitutions": 0, "deletions": 6, "m_wer": 0.6 }, "symptoms": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 12, "correct": 9, "substitutions": 0, "deletions": 3, "m_wer": 0.25 }, "clinical": { "ref_count": 7, "correct": 1, "substitutions": 1, "deletions": 5, "m_wer": 0.857143 } }, "high_risk_m_wer": 1.0, "high_risk_ref_count": 1, "file": "day5_consultation03" }, { "wer": 0.177305, "m_wer": 0.156863, "m_cer": 1.0165, "medical_entity_recall": 0.9259, "medical_token_recall": 0.8627, "total_ref_words": 1269, "total_medical_ref_words": 51, "medical_correct": 44, "medical_substitutions": 7, "medical_deletions": 0, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "substitution", "reference": "implanon", "hypothesis": "on", "category": "drugs", "cer": 0.75, "file": "day5_consultation04" }, { "type": "insertion", "hypothesis": "nauseous", "file": "day5_consultation04" }, { "type": "substitution", "reference": "amoxicillin", "hypothesis": "i", "category": "drugs", "cer": 0.9091, "file": "day5_consultation04" }, { "type": "substitution", "reference": "amoxicillin", "hypothesis": "have", "category": "drugs", "cer": 1.0, "file": "day5_consultation04" }, { "type": "substitution", "reference": "allergy", "hypothesis": "microcellin", "category": "clinical", "cer": 1.4286, "file": "day5_consultation04" }, { "type": "substitution", "reference": "pain", "hypothesis": "bleeding", "category": "symptoms", "cer": 1.5, "file": "day5_consultation04" }, { "type": "substitution", "reference": "discharge", "hypothesis": "other", "category": "symptoms", "cer": 0.7778, "file": "day5_consultation04" }, { "type": "substitution", "reference": "implanon", "hypothesis": "on", "category": "drugs", "cer": 0.75, "file": "day5_consultation04" } ], "unique_medical_terms_in_ref": 27, "unique_medical_terms_found": 25, "per_category": { "drugs": { "ref_count": 4, "correct": 0, "substitutions": 4, "deletions": 0, "m_wer": 1.0 }, "conditions": { "ref_count": 1, "correct": 1, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 25, "correct": 23, "substitutions": 2, "deletions": 0, "m_wer": 0.08 }, "anatomy": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 10, "correct": 9, "substitutions": 1, "deletions": 0, "m_wer": 0.1 } }, "high_risk_m_wer": 1.0, "high_risk_ref_count": 4, "file": "day5_consultation04" }, { "wer": 0.113082, "m_wer": 0.04, "m_cer": 0.5, "medical_entity_recall": 0.9565, "medical_token_recall": 0.96, "total_ref_words": 902, "total_medical_ref_words": 50, "medical_correct": 48, "medical_substitutions": 1, "medical_deletions": 1, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "substitution", "reference": "nausea", "hypothesis": "nauseous", "category": "symptoms", "cer": 0.5, "file": "day5_consultation05" }, { "type": "deletion", "reference": "sore", "category": "symptoms", "file": "day5_consultation05" } ], "unique_medical_terms_in_ref": 23, "unique_medical_terms_found": 22, "per_category": { "drugs": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 24, "correct": 22, "substitutions": 1, "deletions": 1, "m_wer": 0.083333 }, "anatomy": { "ref_count": 11, "correct": 11, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 5, "file": "day5_consultation05" }, { "wer": 0.126061, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 1531, "total_medical_ref_words": 51, "medical_correct": 51, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 26, "per_category": { "drugs": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 21, "correct": 21, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 16, "correct": 16, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 3, "file": "day5_consultation06" }, { "wer": 0.084142, "m_wer": 0.023256, "m_cer": 0.0, "medical_entity_recall": 0.9722, "medical_token_recall": 0.9767, "total_ref_words": 1236, "total_medical_ref_words": 86, "medical_correct": 84, "medical_substitutions": 0, "medical_deletions": 2, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "sinuses", "category": "anatomy", "file": "day5_consultation07" }, { "type": "deletion", "reference": "blood", "category": "anatomy", "file": "day5_consultation07" } ], "unique_medical_terms_in_ref": 36, "unique_medical_terms_found": 35, "per_category": { "drugs": { "ref_count": 7, "correct": 7, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 15, "correct": 15, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 29, "correct": 29, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 21, "correct": 19, "substitutions": 0, "deletions": 2, "m_wer": 0.095238 }, "clinical": { "ref_count": 14, "correct": 14, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 7, "file": "day5_consultation07" }, { "wer": 0.080252, "m_wer": 0.106061, "m_cer": 0.7, "medical_entity_recall": 0.9231, "medical_token_recall": 0.9091, "total_ref_words": 1271, "total_medical_ref_words": 66, "medical_correct": 60, "medical_substitutions": 3, "medical_deletions": 3, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "substitution", "reference": "wheezy", "hypothesis": "dizzy", "category": "symptoms", "cer": 0.6667, "file": "day5_consultation08" }, { "type": "insertion", "hypothesis": "asthma", "file": "day5_consultation08" }, { "type": "substitution", "reference": "eczema", "hypothesis": "well", "category": "conditions", "cer": 0.8333, "file": "day5_consultation08" }, { "type": "substitution", "reference": "cough", "hypothesis": "cold", "category": "symptoms", "cer": 0.6, "file": "day5_consultation08" }, { "type": "deletion", "reference": "cough", "category": "symptoms", "file": "day5_consultation08" }, { "type": "deletion", "reference": "inhaler", "category": "drugs", "file": "day5_consultation08" }, { "type": "deletion", "reference": "fever", "category": "symptoms", "file": "day5_consultation08" } ], "unique_medical_terms_in_ref": 26, "unique_medical_terms_found": 24, "per_category": { "drugs": { "ref_count": 17, "correct": 16, "substitutions": 0, "deletions": 1, "m_wer": 0.058824 }, "conditions": { "ref_count": 14, "correct": 13, "substitutions": 1, "deletions": 0, "m_wer": 0.071429 }, "symptoms": { "ref_count": 17, "correct": 13, "substitutions": 2, "deletions": 2, "m_wer": 0.235294 }, "anatomy": { "ref_count": 4, "correct": 4, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 13, "correct": 13, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.058824, "high_risk_ref_count": 17, "file": "day5_consultation08" }, { "wer": 0.134164, "m_wer": 0.026667, "m_cer": 0.0, "medical_entity_recall": 0.9722, "medical_token_recall": 0.9867, "total_ref_words": 1364, "total_medical_ref_words": 75, "medical_correct": 74, "medical_substitutions": 0, "medical_deletions": 1, "medical_insertions": 1, "spurious_medical_insertions": 1, "medical_errors_detail": [ { "type": "insertion", "hypothesis": "hayfever", "file": "day5_consultation09" }, { "type": "deletion", "reference": "antihistamine", "category": "drugs", "file": "day5_consultation09" } ], "unique_medical_terms_in_ref": 36, "unique_medical_terms_found": 35, "per_category": { "drugs": { "ref_count": 5, "correct": 4, "substitutions": 0, "deletions": 1, "m_wer": 0.2 }, "conditions": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 32, "correct": 32, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 19, "correct": 19, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 9, "correct": 9, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.2, "high_risk_ref_count": 5, "file": "day5_consultation09" }, { "wer": 0.108949, "m_wer": 0.044776, "m_cer": 0.0, "medical_entity_recall": 0.9545, "medical_token_recall": 0.9552, "total_ref_words": 1542, "total_medical_ref_words": 67, "medical_correct": 64, "medical_substitutions": 0, "medical_deletions": 3, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "cough", "category": "symptoms", "file": "day5_consultation10" }, { "type": "deletion", "reference": "phlegm", "category": "symptoms", "file": "day5_consultation10" }, { "type": "deletion", "reference": "prescribe", "category": "clinical", "file": "day5_consultation10" } ], "unique_medical_terms_in_ref": 22, "unique_medical_terms_found": 21, "per_category": { "drugs": { "ref_count": 10, "correct": 10, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 39, "correct": 37, "substitutions": 0, "deletions": 2, "m_wer": 0.051282 }, "anatomy": { "ref_count": 5, "correct": 5, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 6, "substitutions": 0, "deletions": 1, "m_wer": 0.142857 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 10, "file": "day5_consultation10" }, { "wer": 0.177, "m_wer": 0.086957, "m_cer": 0.0, "medical_entity_recall": 0.8696, "medical_token_recall": 0.913, "total_ref_words": 2000, "total_medical_ref_words": 46, "medical_correct": 42, "medical_substitutions": 0, "medical_deletions": 4, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [ { "type": "deletion", "reference": "swelling", "category": "symptoms", "file": "day5_consultation11" }, { "type": "deletion", "reference": "unwell", "category": "symptoms", "file": "day5_consultation11" }, { "type": "deletion", "reference": "medicine", "category": "drugs", "file": "day5_consultation11" }, { "type": "deletion", "reference": "pharmacy", "category": "clinical", "file": "day5_consultation11" } ], "unique_medical_terms_in_ref": 23, "unique_medical_terms_found": 20, "per_category": { "drugs": { "ref_count": 11, "correct": 10, "substitutions": 0, "deletions": 1, "m_wer": 0.090909 }, "conditions": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 18, "correct": 16, "substitutions": 0, "deletions": 2, "m_wer": 0.111111 }, "anatomy": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 7, "correct": 6, "substitutions": 0, "deletions": 1, "m_wer": 0.142857 } }, "high_risk_m_wer": 0.090909, "high_risk_ref_count": 11, "file": "day5_consultation11" }, { "wer": 0.065906, "m_wer": 0.0, "m_cer": 0.0, "medical_entity_recall": 1.0, "medical_token_recall": 1.0, "total_ref_words": 789, "total_medical_ref_words": 42, "medical_correct": 42, "medical_substitutions": 0, "medical_deletions": 0, "medical_insertions": 0, "spurious_medical_insertions": 0, "medical_errors_detail": [], "unique_medical_terms_in_ref": 24, "unique_medical_terms_found": 24, "per_category": { "drugs": { "ref_count": 8, "correct": 8, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "conditions": { "ref_count": 3, "correct": 3, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "symptoms": { "ref_count": 23, "correct": 23, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "anatomy": { "ref_count": 2, "correct": 2, "substitutions": 0, "deletions": 0, "m_wer": 0.0 }, "clinical": { "ref_count": 6, "correct": 6, "substitutions": 0, "deletions": 0, "m_wer": 0.0 } }, "high_risk_m_wer": 0.0, "high_risk_ref_count": 8, "file": "day5_consultation12" } ] }