{ "schema_version": "1.0.0", "release_id": "deepseek-web-citation-audit-country-expansion-2026-08-06", "benchmark_id": "deepseek-web-citation-audit-v2-country-expansion", "benchmark_version": "2.0.0-country-expansion", "summary_ref": "country-expansion-summary.json", "run_conditions": { "surface": "DeepSeek consumer web chat", "visible_mode": "Instant", "web_search": true, "deepthink": false, "execution_date": "2026-08-06", "execution_policy": "One question per fresh chat; first completed response only. For US09 and IN11, an incomplete initial capture was excluded and one identical-prompt rerun supplied the first completed response.", "capture_exclusions": [ { "question_id": "US09", "excluded_capture": "The initial browser capture ended before the answer completed and was excluded as a technical capture failure.", "canonical_record": "One identical-prompt rerun produced the first completed response and is the only US09 record scored.", "protocol_contamination": false }, { "question_id": "IN11", "excluded_capture": "The initial browser capture was technically truncated before completion and was excluded.", "canonical_record": "One identical-prompt rerun produced the completed canonical response and is the only IN11 record scored.", "protocol_contamination": false } ] }, "countries": [ { "country_code": "US", "country_name": "United States", "language": "English", "overall_result": "FAIL", "overall_result_basis": "Descriptive application of the inherited V1 gates; no distinct country-aggregate PASS rule was preregistered for this expansion.", "questions_passed": { "numerator": 4, "denominator": 12, "percent": 33.3333 }, "macro_final_numeric_score": 74.8202, "metrics": { "factual_accuracy": { "numerator": 83, "denominator": 93, "percent": 89.2473 }, "citation_url_validity": { "numerator": 103, "denominator": 115, "percent": 89.5652 }, "citation_support_precision": { "numerator": 147.5, "denominator": 180, "percent": 81.9444 }, "citation_completeness": { "numerator": 70, "denominator": 93, "percent": 75.2688 }, "authority_coverage": { "numerator": 61, "denominator": 93, "percent": 65.5914 }, "citation_placement_precision": { "numerator": 102, "denominator": 115, "percent": 88.6957 } }, "format_score": { "numerator": 32, "denominator": 60, "percent": 53.3333 }, "citation_resolution": { "valid": 103, "invalid": 1, "unverifiable": 11, "fabricated": 0, "total": 115 }, "critical_events": 0, "material_contradiction_questions": [] }, { "country_code": "IN", "country_name": "India", "language": "English", "overall_result": "FAIL", "overall_result_basis": "Descriptive application of the inherited V1 gates; no distinct country-aggregate PASS rule was preregistered for this expansion.", "questions_passed": { "numerator": 0, "denominator": 12, "percent": 0 }, "macro_final_numeric_score": 42.1966, "metrics": { "factual_accuracy": { "numerator": 68, "denominator": 99, "percent": 68.6869 }, "citation_url_validity": { "numerator": 61, "denominator": 80, "percent": 76.25 }, "citation_support_precision": { "numerator": 91.5, "denominator": 129, "percent": 70.9302 }, "citation_completeness": { "numerator": 34, "denominator": 99, "percent": 34.3434 }, "authority_coverage": { "numerator": 25, "denominator": 99, "percent": 25.2525 }, "citation_placement_precision": { "numerator": 60, "denominator": 80, "percent": 75 } }, "format_score": { "numerator": 27, "denominator": 60, "percent": 45 }, "citation_resolution": { "valid": 61, "invalid": 9, "unverifiable": 10, "fabricated": 0, "total": 80 }, "critical_events": 1, "material_contradiction_questions": [ "IN02", "IN04", "IN08", "IN09" ] } ], "questions": [ { "question_id": "US01", "country_code": "US", "country_name": "United States", "title": "Constitutional qualifications for the House and Senate", "category": "constitutional_law", "expected_answer_length": "2-4 sentences", "temporal_risk": "low", "required_claim_count": 6, "required_claim_weight": 6, "metrics": { "factual_accuracy": { "numerator": 6, "denominator": 6, "percent": 100 }, "citation_url_validity": { "numerator": 6, "denominator": 6, "percent": 100 }, "citation_support_precision": { "numerator": 18, "denominator": 18, "percent": 100 }, "citation_completeness": { "numerator": 6, "denominator": 6, "percent": 100 }, "authority_coverage": { "numerator": 6, "denominator": 6, "percent": 100 }, "citation_placement_precision": { "numerator": 6, "denominator": 6, "percent": 100 } }, "citation_resolution": { "valid": 6, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 6 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "Two sentences and precise placement; only the unrequested question-ID heading point is absent." }, "raw_numeric_score": 99, "severity_deduction": 0, "final_numeric_score": 99, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [], "result": "PASS", "key_finding": "All six constitutional qualifications were accurate, complete, and supported by official constitutional sources. Odd indexed suffix variants on two Congress URLs were manually confirmed as substantive official pages.", "official_gold_sources": [ { "source_id": "US-S01A", "title": "Article I, Section 2, Clause 2", "publisher": "Constitution Annotated, Congress.gov, Library of Congress", "url": "https://constitution.congress.gov/browse/article-1/section-2/clause-2", "authority_tier": "A" }, { "source_id": "US-S01B", "title": "Overview of Senate Qualifications Clause", "publisher": "Constitution Annotated, Congress.gov, Library of Congress", "url": "https://constitution.congress.gov/browse/essay/artI-S3-C3-1/ALDE_00013345/", "authority_tier": "A" } ] }, { "question_id": "US02", "country_code": "US", "country_name": "United States", "title": "Copyright duration for post-1977 works", "category": "copyright_law", "expected_answer_length": "2-4 sentences", "temporal_risk": "low", "required_claim_count": 4, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_url_validity": { "numerator": 6, "denominator": 6, "percent": 100 }, "citation_support_precision": { "numerator": 12, "denominator": 12, "percent": 100 }, "citation_completeness": { "numerator": 7, "denominator": 7, "percent": 100 }, "authority_coverage": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_placement_precision": { "numerator": 6, "denominator": 6, "percent": 100 } }, "citation_resolution": { "valid": 6, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 6 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "Two sentences within range; only the unrequested question-ID heading point is absent." }, "raw_numeric_score": 99, "severity_deduction": 0, "final_numeric_score": 99, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [], "result": "PASS", "key_finding": "All post-1977 duration rules, including the earlier-expiring 95-year or 120-year term, were correct and adjacent to a Copyright Office source.", "official_gold_sources": [ { "source_id": "US-S02", "title": "What Is Copyright?", "publisher": "U.S. Copyright Office", "url": "https://copyright.gov/what-is-copyright/", "authority_tier": "A" }, { "source_id": "US-S02B", "title": "Circular 15A: Duration of Copyright", "publisher": "U.S. Copyright Office", "url": "https://www.copyright.gov/circs/circ15a.pdf", "authority_tier": "A" } ] }, { "question_id": "US03", "country_code": "US", "country_name": "United States", "title": "EEOC charge-filing time limits", "category": "employment_law", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 4, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 5, "denominator": 7, "percent": 71.4286 }, "citation_url_validity": { "numerator": 8, "denominator": 9, "percent": 88.8889 }, "citation_support_precision": { "numerator": 10, "denominator": 14, "percent": 71.4286 }, "citation_completeness": { "numerator": 3, "denominator": 7, "percent": 42.8571 }, "authority_coverage": { "numerator": 3, "denominator": 7, "percent": 42.8571 }, "citation_placement_precision": { "numerator": 7, "denominator": 9, "percent": 77.7778 } }, "citation_resolution": { "valid": 8, "invalid": 0, "unverifiable": 1, "fabricated": 0, "total": 9 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": true, "note": "Four sentences are within range, but the ID heading is absent and placement is below 90%." }, "raw_numeric_score": 65.3333, "severity_deduction": 10, "final_numeric_score": 55.3333, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "factual_accuracy_at_least_90", "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "citation_completeness_equals_100", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations" ], "result": "FAIL", "key_finding": "The 180-day, 300-day, and federal 45-day periods were right, but the requested age-discrimination condition was materially missing and one citation was unresolvable.", "official_gold_sources": [ { "source_id": "US-S03", "title": "Time Limits for Filing a Charge", "publisher": "U.S. Equal Employment Opportunity Commission", "url": "https://www.eeoc.gov/time-limits-filing-charge", "authority_tier": "A" } ] }, { "question_id": "US04", "country_code": "US", "country_name": "United States", "title": "DOT automatic airline-refund rule", "category": "transportation_regulation", "expected_answer_length": "3-5 sentences", "temporal_risk": "high", "required_claim_count": 5, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 6, "denominator": 9, "percent": 66.6667 }, "citation_url_validity": { "numerator": 3, "denominator": 4, "percent": 75 }, "citation_support_precision": { "numerator": 7, "denominator": 12, "percent": 58.3333 }, "citation_completeness": { "numerator": 3, "denominator": 9, "percent": 33.3333 }, "authority_coverage": { "numerator": 0.75, "denominator": 9, "percent": 8.3333 }, "citation_placement_precision": { "numerator": 3, "denominator": 4, "percent": 75 } }, "citation_resolution": { "valid": 3, "invalid": 1, "unverifiable": 0, "fabricated": 0, "total": 4 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": false, "note": "The answer used two sentences instead of the required three to five, omitted the ID heading, and placement was below 90%." }, "raw_numeric_score": 53.0833, "severity_deduction": 22, "final_numeric_score": 31.0833, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "factual_accuracy_at_least_90", "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "citation_completeness_equals_100", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations" ], "result": "FAIL", "key_finding": "The payment deadlines were correct, but core trigger qualifiers were omitted, one URL was a soft 404, and no DOT source was cited.", "official_gold_sources": [ { "source_id": "US-S04A", "title": "What Airline Passengers Need to Know About DOT's Automatic Refund Rule", "publisher": "U.S. Department of Transportation", "url": "https://www.transportation.gov/briefing-room/what-airline-passengers-need-know-about-dots-automatic-refund-rule", "authority_tier": "A" }, { "source_id": "US-S04B", "title": "Refunds and Other Consumer Protections - Final Rule", "publisher": "U.S. Department of Transportation", "url": "https://www.transportation.gov/regulations/federal-register-documents/2024-07177", "authority_tier": "A" } ] }, { "question_id": "US05", "country_code": "US", "country_name": "United States", "title": "NIST AI Risk Management Framework Core", "category": "ai_governance", "expected_answer_length": "3-5 sentences", "temporal_risk": "high", "required_claim_count": 6, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 8, "percent": 87.5 }, "citation_url_validity": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_support_precision": { "numerator": 18, "denominator": 18, "percent": 100 }, "citation_completeness": { "numerator": 6, "denominator": 8, "percent": 75 }, "authority_coverage": { "numerator": 6, "denominator": 8, "percent": 75 }, "citation_placement_precision": { "numerator": 9, "denominator": 9, "percent": 100 } }, "citation_resolution": { "valid": 9, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 9 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "Three sentences and precise placement; only the unrequested ID heading point is absent." }, "raw_numeric_score": 88.375, "severity_deduction": 10, "final_numeric_score": 78.375, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "factual_accuracy_at_least_90", "citation_completeness_equals_100", "authority_coverage_at_least_80" ], "result": "FAIL", "key_finding": "The four functions, cross-cutting Govern role, Map, and non-linear framing were right; Measure and Manage omitted required method, prioritization, monitoring, and improvement details.", "official_gold_sources": [ { "source_id": "US-S05A", "title": "AI RMF Core", "publisher": "NIST AI Resource Center", "url": "https://airc.nist.gov/airmf-resources/airmf/5-sec-core/", "authority_tier": "A" }, { "source_id": "US-S05B", "title": "Artificial Intelligence Risk Management Framework (AI RMF 1.0)", "publisher": "National Institute of Standards and Technology", "url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf", "authority_tier": "A" } ] }, { "question_id": "US06", "country_code": "US", "country_name": "United States", "title": "CISA Secure by Design principles", "category": "cybersecurity_policy", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 4, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_url_validity": { "numerator": 12, "denominator": 12, "percent": 100 }, "citation_support_precision": { "numerator": 18, "denominator": 18, "percent": 100 }, "citation_completeness": { "numerator": 7, "denominator": 7, "percent": 100 }, "authority_coverage": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_placement_precision": { "numerator": 12, "denominator": 12, "percent": 100 } }, "citation_resolution": { "valid": 12, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 12 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "Four sentences and precise placement; only the unrequested ID heading point is absent." }, "raw_numeric_score": 99, "severity_deduction": 0, "final_numeric_score": 99, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [], "result": "PASS", "key_finding": "All three CISA principle names and their organizational implications were accurate, complete, and supported by CISA sources.", "official_gold_sources": [ { "source_id": "US-S06", "title": "Applying Secure by Design Thinking to Events in the News", "publisher": "Cybersecurity and Infrastructure Security Agency", "url": "https://www.cisa.gov/news-events/news/applying-secure-design-thinking-events-news", "authority_tier": "A" } ] }, { "question_id": "US07", "country_code": "US", "country_name": "United States", "title": "U.S. Air Quality Index categories", "category": "environmental_health", "expected_answer_length": "one compact table plus 1-2 sentences", "temporal_risk": "medium", "required_claim_count": 7, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 8, "denominator": 8, "percent": 100 }, "citation_url_validity": { "numerator": 16, "denominator": 16, "percent": 100 }, "citation_support_precision": { "numerator": 13, "denominator": 16, "percent": 81.25 }, "citation_completeness": { "numerator": 8, "denominator": 8, "percent": 100 }, "authority_coverage": { "numerator": 8, "denominator": 8, "percent": 100 }, "citation_placement_precision": { "numerator": 16, "denominator": 16, "percent": 100 } }, "citation_resolution": { "valid": 16, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 16 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "The complete table was scored from the DOM snapshot; only the unrequested ID heading point is absent." }, "raw_numeric_score": 95.25, "severity_deduction": 0, "final_numeric_score": 95.25, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "citation_support_precision_at_least_90" ], "result": "FAIL", "key_finding": "Every AQI category and the AQI-100 explanation were correct. Six redundant student-page citations supplied colors and ranges but not the category names, reducing edge-level support below 90%.", "official_gold_sources": [ { "source_id": "US-S07", "title": "Air Quality Index (AQI) Basics", "publisher": "AirNow.gov, U.S. Environmental Protection Agency", "url": "https://www.airnow.gov/aqi/aqi-basics/", "authority_tier": "A" } ] }, { "question_id": "US08", "country_code": "US", "country_name": "United States", "title": "FDA's nine major food allergens", "category": "food_regulation", "expected_answer_length": "2-4 sentences", "temporal_risk": "medium", "required_claim_count": 4, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_url_validity": { "numerator": 4, "denominator": 5, "percent": 80 }, "citation_support_precision": { "numerator": 8.5, "denominator": 14, "percent": 60.7143 }, "citation_completeness": { "numerator": 7, "denominator": 7, "percent": 100 }, "authority_coverage": { "numerator": 1.75, "denominator": 7, "percent": 25 }, "citation_placement_precision": { "numerator": 4, "denominator": 5, "percent": 80 } }, "citation_resolution": { "valid": 4, "invalid": 0, "unverifiable": 1, "fabricated": 0, "total": 5 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": true, "note": "Two sentences are within range, but the ID heading is absent and placement is below 90%." }, "raw_numeric_score": 78.6429, "severity_deduction": 0, "final_numeric_score": 78.6429, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations" ], "result": "FAIL", "key_finding": "The facts were correct and fully cited, but the sources were commercial rather than FDA, one social citation was unresolvable, and several citation-to-claim edges were only partial.", "official_gold_sources": [ { "source_id": "US-S08", "title": "The FASTER Act: Sesame Is the Ninth Major Food Allergen", "publisher": "U.S. Food and Drug Administration", "url": "https://www.fda.gov/food/food-allergies/faster-act-sesame-ninth-major-food-allergen", "authority_tier": "A" } ] }, { "question_id": "US09", "country_code": "US", "country_name": "United States", "title": "USDA safe minimum internal temperatures", "category": "food_safety", "expected_answer_length": "one compact table", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 8, "denominator": 9, "percent": 88.8889 }, "citation_url_validity": { "numerator": 17, "denominator": 19, "percent": 89.4737 }, "citation_support_precision": { "numerator": 14.5, "denominator": 21, "percent": 69.0476 }, "citation_completeness": { "numerator": 4, "denominator": 9, "percent": 44.4444 }, "authority_coverage": { "numerator": 4, "denominator": 9, "percent": 44.4444 }, "citation_placement_precision": { "numerator": 17, "denominator": 19, "percent": 89.4737 } }, "citation_resolution": { "valid": 17, "invalid": 0, "unverifiable": 2, "fabricated": 0, "total": 19 }, "format": { "points": 0, "possible": 5, "percent": 0, "prompt_contract_compliant": false, "note": "The prompt required one compact table, but the answer added a preface and note; it also lacked the ID heading and placement was below 90%." }, "raw_numeric_score": 69.4528, "severity_deduction": 12, "final_numeric_score": 57.4528, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "factual_accuracy_at_least_90", "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "citation_completeness_equals_100", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations" ], "result": "FAIL", "key_finding": "The completed rerun table had the main temperatures, but omitted stuffing, contained two unresolvable FSIS citation occurrences, and often cited sources that supplied only Fahrenheit or rounded Celsius values. Uncited 'Not required' cells did not earn same-cell completeness credit.", "official_gold_sources": [ { "source_id": "US-S09", "title": "Safe Minimum Internal Temperature Chart", "publisher": "USDA Food Safety and Inspection Service", "url": "https://www.fsis.usda.gov/food-safety/safe-food-handling-and-preparation/food-safety-basics/safe-temperature-chart", "authority_tier": "A" } ] }, { "question_id": "US10", "country_code": "US", "country_name": "United States", "title": "Series EE savings-bond structural rules", "category": "public_finance", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_url_validity": { "numerator": 8, "denominator": 9, "percent": 88.8889 }, "citation_support_precision": { "numerator": 9.5, "denominator": 11, "percent": 86.3636 }, "citation_completeness": { "numerator": 7, "denominator": 7, "percent": 100 }, "authority_coverage": { "numerator": 5.5, "denominator": 7, "percent": 78.5714 }, "citation_placement_precision": { "numerator": 8, "denominator": 9, "percent": 88.8889 } }, "citation_resolution": { "valid": 8, "invalid": 0, "unverifiable": 1, "fabricated": 0, "total": 9 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": true, "note": "Three sentences are within range, but the ID heading is absent and placement is below 90%." }, "raw_numeric_score": 90.4632, "severity_deduction": 0, "final_numeric_score": 90.4632, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations" ], "result": "FAIL", "key_finding": "All five bond rules were correct, but one legacy government-looking PDF could not be resolved, some support was partial, and the only fully supporting adjacent source for the early-redemption penalty was commercial Tier C.", "official_gold_sources": [ { "source_id": "US-S10", "title": "EE Bonds", "publisher": "U.S. Department of the Treasury, TreasuryDirect", "url": "https://www.treasurydirect.gov/savings-bonds/ee-bonds/", "authority_tier": "A" } ] }, { "question_id": "US11", "country_code": "US", "country_name": "United States", "title": "Earthquake magnitude versus intensity", "category": "earth_science", "expected_answer_length": "3-5 sentences", "temporal_risk": "low", "required_claim_count": 5, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_url_validity": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_support_precision": { "numerator": 14, "denominator": 15, "percent": 93.3333 }, "citation_completeness": { "numerator": 9, "denominator": 9, "percent": 100 }, "authority_coverage": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_placement_precision": { "numerator": 9, "denominator": 9, "percent": 100 } }, "citation_resolution": { "valid": 9, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 9 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": true, "note": "Three sentences and precise placement; only the unrequested ID heading point is absent." }, "raw_numeric_score": 97.6667, "severity_deduction": 0, "final_numeric_score": 97.6667, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [], "result": "PASS", "key_finding": "Magnitude, intensity, MMI, Moment Magnitude, and the large-teleseismic Richter qualification were all accurate and supported by USGS sources.", "official_gold_sources": [ { "source_id": "US-S11A", "title": "What is the difference between earthquake magnitude and earthquake intensity?", "publisher": "U.S. Geological Survey", "url": "https://www.usgs.gov/faqs/what-difference-between-earthquake-magnitude-and-earthquake-intensity-what-modified-mercalli", "authority_tier": "A" }, { "source_id": "US-S11B", "title": "How are earthquakes recorded and measured?", "publisher": "U.S. Geological Survey", "url": "https://www.usgs.gov/faqs/how-are-earthquakes-recorded-how-are-earthquakes-measured-how-magnitude-earthquake-determined", "authority_tier": "A" } ] }, { "question_id": "US12", "country_code": "US", "country_name": "United States", "title": "NFIP flood-insurance waiting period and exceptions", "category": "disaster_insurance", "expected_answer_length": "4-6 sentences", "temporal_risk": "high", "required_claim_count": 5, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 6, "denominator": 9, "percent": 66.6667 }, "citation_url_validity": { "numerator": 5, "denominator": 11, "percent": 45.4545 }, "citation_support_precision": { "numerator": 5, "denominator": 11, "percent": 45.4545 }, "citation_completeness": { "numerator": 3, "denominator": 9, "percent": 33.3333 }, "authority_coverage": { "numerator": 3, "denominator": 9, "percent": 33.3333 }, "citation_placement_precision": { "numerator": 5, "denominator": 11, "percent": 45.4545 } }, "citation_resolution": { "valid": 5, "invalid": 0, "unverifiable": 6, "fabricated": 0, "total": 11 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": false, "note": "The answer used two sentences instead of four to six, omitted the ID heading, and placement was below 90%." }, "raw_numeric_score": 48.5758, "severity_deduction": 32, "final_numeric_score": 16.5758, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_numeric_score_at_least_85", "factual_accuracy_at_least_90", "citation_url_validity_equals_100", "citation_support_precision_at_least_90", "citation_completeness_equals_100", "authority_coverage_at_least_80", "zero_invalid_or_unresolvable_citations", "zero_material_contradiction_of_a_required_claim" ], "result": "FAIL", "key_finding": "The default period and renewal exception were correct, but the mortgage scope, 12-month map window, and one-day wildfire rule did not match the named current consumer page. Six dated FEMA citation occurrences could not be substantiated.", "official_gold_sources": [ { "source_id": "US-S12", "title": "Buy a Flood Insurance Policy", "publisher": "National Flood Insurance Program, FloodSmart.gov", "url": "https://www.floodsmart.gov/get-insured/buy-a-policy", "authority_tier": "A" } ] }, { "question_id": "IN01", "country_code": "IN", "country_name": "India", "title": "Right to Information Act request and response deadlines", "category": "access_to_information_law", "expected_answer_length": "3-5 sentences", "temporal_risk": "low", "required_claim_count": 5, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 8, "denominator": 8, "percent": 100 }, "citation_url_validity": { "numerator": 0, "denominator": 3, "percent": 0 }, "citation_support_precision": { "numerator": 0, "denominator": 5, "percent": 0 }, "citation_completeness": { "numerator": 0, "denominator": 8, "percent": 0 }, "authority_coverage": { "numerator": 0, "denominator": 8, "percent": 0 }, "citation_placement_precision": { "numerator": 0, "denominator": 3, "percent": 0 } }, "citation_resolution": { "valid": 0, "invalid": 0, "unverifiable": 3, "fabricated": 0, "total": 3 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": false, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 36, "severity_deduction": 2, "final_numeric_score": 34, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations" ], "result": "FAIL", "key_finding": "All five RTI rules were factually correct, but all three citations pointed to one uninspectable LinkedIn post rather than India Code.", "official_gold_sources": [ { "source_id": "IN-S01A", "title": "Right to Information Act, 2005 - Section 6", "publisher": "India Code, Legislative Department", "url": "https://www.indiacode.nic.in/show-data?actid=AC_CEN_26_36_00004_200522_1517807322955&orderno=6", "authority_tier": "A" }, { "source_id": "IN-S01B", "title": "Right to Information Act, 2005 - Section 7", "publisher": "India Code, Legislative Department", "url": "https://www.indiacode.nic.in/show-data?actid=AC_CEN_26_36_00004_200522_1517807322955&orderno=7", "authority_tier": "A" } ] }, { "question_id": "IN02", "country_code": "IN", "country_name": "India", "title": "E-commerce grievance, cancellation, and consent rules", "category": "consumer_protection_law", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 6, "denominator": 8, "percent": 75 }, "citation_url_validity": { "numerator": 3, "denominator": 5, "percent": 60 }, "citation_support_precision": { "numerator": 4, "denominator": 9, "percent": 44.4444 }, "citation_completeness": { "numerator": 6, "denominator": 8, "percent": 75 }, "authority_coverage": { "numerator": 0, "denominator": 8, "percent": 0 }, "citation_placement_precision": { "numerator": 2, "denominator": 5, "percent": 40 } }, "citation_resolution": { "valid": 3, "invalid": 0, "unverifiable": 2, "fabricated": 0, "total": 5 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 57.3889, "severity_deduction": 10, "final_numeric_score": 47.3889, "critical_event_count": 0, "material_contradiction": true, "failed_hard_gates": [ "final_score", "factual_accuracy", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations", "zero_material_contradictions" ], "result": "FAIL", "key_finding": "The complaint deadlines and consent rule were right. The cancellation-charge condition materially contradicted the final E-Commerce Rules, and no official source was cited.", "official_gold_sources": [ { "source_id": "IN-S02", "title": "Consumer Protection (E-Commerce) Rules, 2020", "publisher": "Department of Consumer Affairs, Government of India", "url": "https://consumeraffairs.nic.in/sites/default/files/E%20commerce%20rules_0.pdf", "authority_tier": "A" } ] }, { "question_id": "IN03", "country_code": "IN", "country_name": "India", "title": "DICGC deposit-insurance coverage", "category": "banking_regulation", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 9, "percent": 77.7778 }, "citation_url_validity": { "numerator": 9, "denominator": 12, "percent": 75 }, "citation_support_precision": { "numerator": 10, "denominator": 15, "percent": 66.6667 }, "citation_completeness": { "numerator": 6, "denominator": 9, "percent": 66.6667 }, "authority_coverage": { "numerator": 6, "denominator": 9, "percent": 66.6667 }, "citation_placement_precision": { "numerator": 9, "denominator": 12, "percent": 75 } }, "citation_resolution": { "valid": 9, "invalid": 0, "unverifiable": 3, "fabricated": 0, "total": 12 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 70.4722, "severity_deduction": 10, "final_numeric_score": 60.4722, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "factual_accuracy", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations" ], "result": "FAIL", "key_finding": "The amount, interest aggregation, branch aggregation, and separate-bank treatment were right. The ownership rule was incomplete and three raw citation targets could not be verified.", "official_gold_sources": [ { "source_id": "IN-S03", "title": "Frequently Asked Questions", "publisher": "Deposit Insurance and Credit Guarantee Corporation", "url": "https://www.dicgc.org.in/FAQs", "authority_tier": "A" } ] }, { "question_id": "IN04", "country_code": "IN", "country_name": "India", "title": "Aadhaar authentication modes", "category": "digital_identity", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 5, "denominator": 8, "percent": 62.5 }, "citation_url_validity": { "numerator": 0, "denominator": 0, "percent": null }, "citation_support_precision": { "numerator": 0, "denominator": 0, "percent": null }, "citation_completeness": { "numerator": 0, "denominator": 8, "percent": 0 }, "authority_coverage": { "numerator": 0, "denominator": 8, "percent": 0 }, "citation_placement_precision": { "numerator": 0, "denominator": 0, "percent": null } }, "citation_resolution": { "valid": 0, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 0 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": false, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 22.875, "severity_deduction": 42, "final_numeric_score": 0, "critical_event_count": 1, "material_contradiction": true, "failed_hard_gates": [ "final_score", "factual_accuracy", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_critical_events", "zero_material_contradictions" ], "result": "FAIL", "key_finding": "The response contained no citations and invented a prohibition on biometric e-KYC that directly reverses UIDAI's current rule.", "official_gold_sources": [ { "source_id": "IN-S04", "title": "Authentication Ecosystem", "publisher": "Unique Identification Authority of India", "url": "https://uidai.gov.in/en/ecosystem/authentication-ecosystem.html", "authority_tier": "A" } ] }, { "question_id": "IN05", "country_code": "IN", "country_name": "India", "title": "General-voter registration requirements", "category": "election_administration", "expected_answer_length": "2-4 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 6, "metrics": { "factual_accuracy": { "numerator": 3, "denominator": 6, "percent": 50 }, "citation_url_validity": { "numerator": 0, "denominator": 2, "percent": 0 }, "citation_support_precision": { "numerator": 0, "denominator": 2, "percent": 0 }, "citation_completeness": { "numerator": 0, "denominator": 6, "percent": 0 }, "authority_coverage": { "numerator": 0, "denominator": 6, "percent": 0 }, "citation_placement_precision": { "numerator": 0, "denominator": 2, "percent": 0 } }, "citation_resolution": { "valid": 0, "invalid": 0, "unverifiable": 2, "fabricated": 0, "total": 2 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": false, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 18.5, "severity_deduction": 12, "final_numeric_score": 6.5, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "factual_accuracy", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations" ], "result": "FAIL", "key_finding": "The age condition and four dates were right, but citizenship, ordinary residence, and absence of disqualification were omitted; both citations were uninspectable LinkedIn targets.", "official_gold_sources": [ { "source_id": "IN-S05", "title": "How to Register to Vote", "publisher": "Election Commission of India", "url": "https://www.eci.gov.in/faq/en/how-to-register", "authority_tier": "A" } ] }, { "question_id": "IN06", "country_code": "IN", "country_name": "India", "title": "Chandrayaan-3 dates, vehicle, and mission objectives", "category": "space_science", "expected_answer_length": "3-5 sentences", "temporal_risk": "low", "required_claim_count": 4, "required_claim_weight": 9, "metrics": { "factual_accuracy": { "numerator": 8, "denominator": 9, "percent": 88.8889 }, "citation_url_validity": { "numerator": 11, "denominator": 11, "percent": 100 }, "citation_support_precision": { "numerator": 11, "denominator": 11, "percent": 100 }, "citation_completeness": { "numerator": 7, "denominator": 9, "percent": 77.7778 }, "authority_coverage": { "numerator": 7, "denominator": 9, "percent": 77.7778 }, "citation_placement_precision": { "numerator": 11, "denominator": 11, "percent": 100 } }, "citation_resolution": { "valid": 11, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 11 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": true, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 89.5556, "severity_deduction": 10, "final_numeric_score": 79.5556, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "factual_accuracy", "completeness", "authority_coverage" ], "result": "FAIL", "key_finding": "This was the strongest India answer: every citation resolved and supported its sentence, but the exact LVM3-M4 designation was missing.", "official_gold_sources": [ { "source_id": "IN-S06A", "title": "Chandrayaan-3 Details", "publisher": "Indian Space Research Organisation", "url": "https://www.isro.gov.in/ISRO_EN/Chandrayaan3_Details.html", "authority_tier": "A" }, { "source_id": "IN-S06B", "title": "Chandrayaan-3", "publisher": "Indian Space Research Organisation", "url": "https://www.isro.gov.in/ISRO_EN/Chandrayaan3.html", "authority_tier": "A" }, { "source_id": "IN-S06C", "title": "Chandrayaan-3 Propulsion Module moved from lunar orbit to Earth orbit", "publisher": "Indian Space Research Organisation", "url": "https://www.isro.gov.in/ISRO_EN/Ch3_Propulsion_Module_moved_from_Lunar_orbit_to_Earth_orbit.html", "authority_tier": "A" } ] }, { "question_id": "IN07", "country_code": "IN", "country_name": "India", "title": "All India Tiger Estimation 2022", "category": "wildlife_statistics", "expected_answer_length": "3-5 sentences", "temporal_risk": "low", "required_claim_count": 5, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_url_validity": { "numerator": 5, "denominator": 9, "percent": 55.5556 }, "citation_support_precision": { "numerator": 11.5, "denominator": 18, "percent": 63.8889 }, "citation_completeness": { "numerator": 5, "denominator": 7, "percent": 71.4286 }, "authority_coverage": { "numerator": 4.25, "denominator": 7, "percent": 60.7143 }, "citation_placement_precision": { "numerator": 5, "denominator": 9, "percent": 55.5556 } }, "citation_resolution": { "valid": 5, "invalid": 4, "unverifiable": 0, "fabricated": 0, "total": 9 }, "format": { "points": 2, "possible": 5, "percent": 40, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 74.8968, "severity_deduction": 0, "final_numeric_score": 74.8968, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations" ], "result": "FAIL", "key_finding": "All five tiger-estimation facts were right, including 3,682 and the sampling qualifier. Four occurrences used an unresolved host, and the remaining official growth-rate citation did not state the qualifier.", "official_gold_sources": [ { "source_id": "IN-S07A", "title": "All India Tiger Estimation 2022: Release of the Detailed Report", "publisher": "Press Information Bureau, Ministry of Environment, Forest and Climate Change", "url": "https://www.pib.gov.in/PressReleaseIframePage.aspx?PRID=1943922&lang=2®=48", "authority_tier": "A" }, { "source_id": "IN-S07B", "title": "All India Tiger Estimation 2022 official summary", "publisher": "Press Information Bureau", "url": "https://www.pib.gov.in/PressReleasePage.aspx?PRID=1991620&lang=2®=3", "authority_tier": "A" } ] }, { "question_id": "IN08", "country_code": "IN", "country_name": "India", "title": "Census 2011 final population by sex", "category": "official_statistics", "expected_answer_length": "2-4 sentences", "temporal_risk": "low", "required_claim_count": 4, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 0, "denominator": 8, "percent": 0 }, "citation_url_validity": { "numerator": 3, "denominator": 3, "percent": 100 }, "citation_support_precision": { "numerator": 7, "denominator": 7, "percent": 100 }, "citation_completeness": { "numerator": 0, "denominator": 8, "percent": 0 }, "authority_coverage": { "numerator": 0, "denominator": 8, "percent": 0 }, "citation_placement_precision": { "numerator": 3, "denominator": 3, "percent": 100 } }, "citation_resolution": { "valid": 3, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 3 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": true, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 39, "severity_deduction": 10, "final_numeric_score": 29, "critical_event_count": 0, "material_contradiction": true, "failed_hard_gates": [ "final_score", "factual_accuracy", "completeness", "authority_coverage", "zero_material_contradictions" ], "result": "FAIL", "key_finding": "The arithmetic was internally consistent, but every requested figure differed from the frozen final PCA values: 1,210,854,977 total; 623,270,258 male; 587,584,719 female.", "official_gold_sources": [ { "source_id": "IN-S08A", "title": "Population Finder 2011", "publisher": "Office of the Registrar General and Census Commissioner, India", "url": "https://censusindia.gov.in/census.website/en/data/population-finder", "authority_tier": "A" }, { "source_id": "IN-S08B", "title": "Basic Population Figures of India, State, District, Sub-District and Village - 2011", "publisher": "Census of India Microdata Library", "url": "https://censusindia.gov.in/nada/index.php/catalog/42554", "authority_tier": "A" } ] }, { "question_id": "IN09", "country_code": "IN", "country_name": "India", "title": "IMD April 2026 southwest-monsoon forecast table", "category": "climate_forecasting", "expected_answer_length": "4-6 sentences or one compact table plus 1-2 sentences", "temporal_risk": "low", "required_claim_count": 4, "required_claim_weight": 8, "metrics": { "factual_accuracy": { "numerator": 3, "denominator": 8, "percent": 37.5 }, "citation_url_validity": { "numerator": 8, "denominator": 10, "percent": 80 }, "citation_support_precision": { "numerator": 10, "denominator": 14, "percent": 71.4286 }, "citation_completeness": { "numerator": 2, "denominator": 8, "percent": 25 }, "authority_coverage": { "numerator": 0.5, "denominator": 8, "percent": 6.25 }, "citation_placement_precision": { "numerator": 8, "denominator": 10, "percent": 80 } }, "citation_resolution": { "valid": 8, "invalid": 2, "unverifiable": 0, "fabricated": 0, "total": 10 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": false, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 44.7857, "severity_deduction": 22, "final_numeric_score": 22.7857, "critical_event_count": 0, "material_contradiction": true, "failed_hard_gates": [ "final_score", "factual_accuracy", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations", "zero_material_contradictions" ], "result": "FAIL", "key_finding": "The answer used secondary reports, misstated the Below Normal band, and omitted the +/-5% model error and all five probabilities even though the official IMD release contained them.", "official_gold_sources": [ { "source_id": "IN-S09", "title": "Long Range Forecast for Southwest Monsoon Season Rainfall, April 13, 2026", "publisher": "India Meteorological Department", "url": "https://mausam.imd.gov.in/Forecast/marquee_data/Long%20Range%20Forecast%20for%20Southwest%20Monsoon%20Season%20Rainfall_Press_release_13.4.2026%20%281%29.pdf", "authority_tier": "A" } ] }, { "question_id": "IN10", "country_code": "IN", "country_name": "India", "title": "BIS gold-jewellery hallmark components", "category": "product_standards", "expected_answer_length": "3-5 sentences", "temporal_risk": "medium", "required_claim_count": 5, "required_claim_weight": 7, "metrics": { "factual_accuracy": { "numerator": 5, "denominator": 7, "percent": 71.4286 }, "citation_url_validity": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_support_precision": { "numerator": 19, "denominator": 19, "percent": 100 }, "citation_completeness": { "numerator": 3, "denominator": 7, "percent": 42.8571 }, "authority_coverage": { "numerator": 3, "denominator": 7, "percent": 42.8571 }, "citation_placement_precision": { "numerator": 9, "denominator": 9, "percent": 100 } }, "citation_resolution": { "valid": 9, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 9 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": true, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 74.7143, "severity_deduction": 10, "final_numeric_score": 64.7143, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "factual_accuracy", "completeness", "authority_coverage" ], "result": "FAIL", "key_finding": "All nine links resolved and supported the written claims. Missing date and traceability qualifiers prevented the two HUID claims from earning completeness credit.", "official_gold_sources": [ { "source_id": "IN-S10A", "title": "Hallmarking Frequently Asked Questions - General", "publisher": "Bureau of Indian Standards", "url": "https://www.bis.gov.in/hallmarking-overview/hallmarking-faqs/hallmarking-faq/?lang=en", "authority_tier": "A" }, { "source_id": "IN-S10B", "title": "BIS Care App", "publisher": "Bureau of Indian Standards", "url": "https://www.bis.gov.in/bis-apps/?lang=en", "authority_tier": "A" } ] }, { "question_id": "IN11", "country_code": "IN", "country_name": "India", "title": "FSSAI packaged-food allergen declaration", "category": "food_labelling_regulation", "expected_answer_length": "one compact list plus 1-2 sentences", "temporal_risk": "high", "required_claim_count": 5, "required_claim_weight": 10, "metrics": { "factual_accuracy": { "numerator": 9, "denominator": 10, "percent": 90 }, "citation_url_validity": { "numerator": 4, "denominator": 7, "percent": 57.1429 }, "citation_support_precision": { "numerator": 5, "denominator": 9, "percent": 55.5556 }, "citation_completeness": { "numerator": 4, "denominator": 10, "percent": 40 }, "authority_coverage": { "numerator": 4, "denominator": 10, "percent": 40 }, "citation_placement_precision": { "numerator": 4, "denominator": 7, "percent": 57.1429 } }, "citation_resolution": { "valid": 4, "invalid": 3, "unverifiable": 0, "fabricated": 0, "total": 7 }, "format": { "points": 1, "possible": 5, "percent": 20, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": false, "citation_placement_at_least_90_percent": false, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 62.1825, "severity_deduction": 22, "final_numeric_score": 40.1825, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "url_validity", "support_precision", "completeness", "authority_coverage", "zero_unresolvable_citations" ], "result": "FAIL", "key_finding": "The completed canonical rerun was scored. It gave nearly all requested facts, but omitted the cereal detail, used obsolete compendia, left most list items without adjacent citations, and repeated a soft landing-page target three times.", "official_gold_sources": [ { "source_id": "IN-S11A", "title": "Food Safety and Standards (Labelling and Display) Regulations, 2020 - Compendium Version VIII", "publisher": "Food Safety and Standards Authority of India", "url": "https://www.fssai.gov.in/upload/uploadfiles/files/Comp_Labelling%20Display_Version%20VIII_09_09_2025.pdf", "authority_tier": "A" }, { "source_id": "IN-S11B", "title": "Food Safety and Standards Regulations index", "publisher": "Food Safety and Standards Authority of India", "url": "https://fssai.gov.in/cms/food-safety-and-standards-regulations.php", "authority_tier": "A" } ] }, { "question_id": "IN12", "country_code": "IN", "country_name": "India", "title": "Phased commencement of the DPDP Act", "category": "data_protection_law", "expected_answer_length": "5-7 sentences", "temporal_risk": "high", "required_claim_count": 5, "required_claim_weight": 11, "metrics": { "factual_accuracy": { "numerator": 7, "denominator": 11, "percent": 63.6364 }, "citation_url_validity": { "numerator": 9, "denominator": 9, "percent": 100 }, "citation_support_precision": { "numerator": 14, "denominator": 20, "percent": 70 }, "citation_completeness": { "numerator": 1, "denominator": 11, "percent": 9.0909 }, "authority_coverage": { "numerator": 0.25, "denominator": 11, "percent": 2.2727 }, "citation_placement_precision": { "numerator": 9, "denominator": 9, "percent": 100 } }, "citation_resolution": { "valid": 9, "invalid": 0, "unverifiable": 0, "fabricated": 0, "total": 9 }, "format": { "points": 4, "possible": 5, "percent": 80, "prompt_contract_compliant": null, "exact_question_id_heading_present": false, "requested_length_met": true, "no_forbidden_extra": true, "citation_placement_at_least_90_percent": true, "note": "The inherited exact question-ID heading component is retained for score fidelity even though the public prompt did not request that heading; no separate heading severity deduction was applied." }, "raw_numeric_score": 56.8636, "severity_deduction": 10, "final_numeric_score": 46.8636, "critical_event_count": 0, "material_contradiction": false, "failed_hard_gates": [ "final_score", "factual_accuracy", "support_precision", "completeness", "authority_coverage" ], "result": "FAIL", "key_finding": "The three dates and broad phase themes were right, but the answer did not provide the exact Gazette section groups and cited only non-official replicas or legal explainers.", "official_gold_sources": [ { "source_id": "IN-S12A", "title": "G.S.R. 843(E): Enforcement Timeline for the DPDP Act", "publisher": "Ministry of Electronics and Information Technology, Gazette of India", "url": "https://www.meity.gov.in/static/uploads/2025/11/c56ceae6c383460ca69577428d36828b.pdf", "authority_tier": "A" }, { "source_id": "IN-S12B", "title": "Digital Personal Data Protection Act, 2023", "publisher": "India Code, Legislative Department", "url": "https://www.indiacode.nic.in/handle/123456789/22037", "authority_tier": "A" } ] } ] }