import copy import json from decimal import Decimal import jsonschema from .common import ROOT, read, require from .scoring import score def obj(props, required=None): return {'type':'object','properties':props,'required':required or list(props),'additionalProperties':False} def arr(item): return {'type':'array','items':item} S={'type':'string','minLength':1};B={'type':'boolean'};STRINGS=arr(S) POS=obj({'id':S,'verdict':B,'reason':S,'quotes':STRINGS,'unresolved_contradictions':STRINGS}) NEG=obj({'id':S,'verdict':B,'reason':S,'quotes':STRINGS,'core_dependency':B,'dependency_chain':S,'counterfactual':S}) PROOF=obj({'original_threshold':{'type':'number'},'corrected_threshold':{'type':'number'},'test_value':{'type':'number'},'operator':{'enum':['gt','ge','lt','le']},'unit':S,'source_value':S}) ERROR=obj({'id':S,'claim_ids':STRINGS,'original_negative_id':{'type':['string','null']}, 'severity':{'enum':['MINOR','MATERIAL','CORE']},'reason':S,'correction_unit':S,'quotes':STRINGS, 'scope_positive_ids':STRINGS,'affected_positive_ids':STRINGS,'causal_credit_positive_ids':STRINGS, 'causal_credit_reason':S,'dependency_chain':S,'counterfactual':S,'unaffected_conclusions':STRINGS, 'role':{'enum':['decision_threshold','calculation_input','supporting_detail','no_downstream_use']}, 'threshold_proof':{'anyOf':[{'type':'null'},PROOF]}}) SCHEMA=obj({'status':{'enum':['complete','incomplete']},'missing_requirements':arr(S), 'positive':arr(POS),'negative':arr(NEG), 'fact_reviews':arr(obj({'claim_id':S,'actionable':B,'error_id':{'type':['string','null']},'reason':S,'quotes':STRINGS})), 'fact_errors':arr(ERROR)}) def evaluate(raw, rubric, answer, audit): jsonschema.validate(raw,SCHEMA) require(raw['status']=='complete' and not raw['missing_requirements'],'Quality judgment incomplete') ps={p['id']:p for p in rubric['positive']};ns={n['id']:n for n in rubric['negative']} for typ,expected in [('positive',ps),('negative',ns)]: require(len(raw[typ])==len(expected) and {x['id'] for x in raw[typ]}==set(expected),'Missing/duplicate rubric judgments') def quotes(x, mandatory=False): require(not mandatory or bool(x['quotes']),'Missing answer witness') require(all(q in answer for q in x['quotes']),'Quote is not verbatim answer text') pv={p['id']:p['verdict'] for p in raw['positive']} nv={n['id']:n['verdict'] for n in raw['negative']} for p in raw['positive']: quotes(p) require(all(q in answer for q in p['unresolved_contradictions']),'Invented contradiction witness') require(not p['unresolved_contradictions'] or not p['verdict'],'Unresolved contradiction cannot pass') effective=copy.deepcopy(rubric) en={n['id']:n for n in effective['negative']} for n in raw['negative']: quotes(n,n['verdict']) require(not n['core_dependency'] or n['verdict'],'Untriggered negative cannot be core') if n['core_dependency']:en[n['id']]['score_cap']=min(en[n['id']]['score_cap'] or 100,60) eligible={a['claim_id']:a for a in audit['items'] if a['factual_verdict']=='contradicted'} reviews={f['claim_id']:f for f in raw['fact_reviews']} require(len(reviews)==len(raw['fact_reviews']) and reviews.keys()==eligible.keys(),'Every factual contradiction needs review, including Trace-supported conflicts') errors={e['id']:e for e in raw['fact_errors']} require(len(errors)==len(raw['fact_errors']) and not (set(errors)&(set(ps)|set(ns))),'Duplicate error IDs') roots=set();claimed=set();credits=set();extras=[] for f in reviews.values(): quotes(f,True) require((f['error_id'] in errors) if f['actionable'] else f['error_id'] is None,'Invalid fact disposition') for eid,e in errors.items(): quotes(e,True) require(e['correction_unit'] not in roots,'Duplicate correction unit');roots.add(e['correction_unit']) require(bool(e['claim_ids']) and len(set(e['claim_ids']))==len(e['claim_ids']) and set(e['claim_ids'])<=eligible.keys(),'Invalid fact claim IDs') require(not claimed.intersection(e['claim_ids']),'Fact counted more than once');claimed.update(e['claim_ids']) require(all(reviews[c]['actionable'] and reviews[c]['error_id']==eid for c in e['claim_ids']),'Review/error mismatch') for key in ['scope_positive_ids','affected_positive_ids','causal_credit_positive_ids']: require(len(set(e[key]))==len(e[key]) and set(e[key])<=ps.keys(),'Invalid positive links') scope=set(e['scope_positive_ids']);affected=set(e['affected_positive_ids']);credit=set(e['causal_credit_positive_ids']) require(credit<=affected<=scope,'Credit/affected/scope mismatch') require(not any(pv[k] for k in affected),'Affected full positive predicate still passes') require(not credits.intersection(credit),'Duplicate positive loss credit');credits.update(credit) require((not scope) if e['severity']=='MINOR' else bool(scope),'Severity/task scope mismatch') require(bool(e['unaffected_conclusions']),'Missing unaffected-path assessment') if e['role']=='decision_threshold': p=e['threshold_proof'];require(p is not None,'Threshold proof missing') require(any(p['source_value'] in q for q in e['quotes']),'Threshold not grounded in source quote') require(Decimal(p['source_value'])==Decimal(str(p['original_threshold'])),'Invented original threshold') compare={'gt':lambda a,b:a>b,'ge':lambda a,b:a>=b,'lt':lambda a,b:a