#!/usr/bin/env python3 # Copyright (C) 2026 Yusuke Oda # SPDX-License-Identifier: AGPL-3.0-only """Read-only audit of preregistered confirmation results, without recomputing tests.""" import argparse import datetime import hashlib import json import math import random from pathlib import Path ROOT = Path(__file__).resolve().parent.parent FINGERPRINT_FIELDS = ('case_id', 'frames', 'channels', 'checksum', 'peak', 'rms', 'state_bytes', 'latency_samples') OPTIONAL_FIELDS = ('effective_controls', 'observed_frames', 'output_buffer_frames', 'scalar_bytes') def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() def audit_structure(result, errors, label): plan = result['plan'] jobs = plan['jobs'] identifiers = [job['id'] for job in jobs] if len(set(identifiers)) != len(identifiers): errors.append(label + ': duplicate registered job IDs') if result.get('status') != 'complete': errors.append(label + ': result is not complete') blocks = result['blocks'] if len(blocks) != plan['blocks']: errors.append(label + ': completed block count differs from plan') if len(result['orders']) != plan['blocks']: errors.append(label + ': random-order count differs from plan') rng = random.Random(plan['seed']) expected_orders = [] for _ in range(plan['blocks']): order = list(range(len(jobs))) rng.shuffle(order) expected_orders.append(order) if result['orders'] != expected_orders: errors.append(label + ': stored permutations differ from the registered random seed') job_map = {job['id']: job for job in jobs} fingerprints = {} minimum_burn = None minimum_gap = None runs_seen = 0 previous_block = None for index, block in enumerate(blocks): if block['number'] != index + 1: errors.append(label + ': block numbering is not consecutive') if previous_block is not None: gap = block['started_ns'] - previous_block['finished_ns'] minimum_gap = gap if minimum_gap is None else min(minimum_gap, gap) if gap < plan['cooldown_seconds'] * 1_000_000_000: errors.append(label + ': block cooldown below requested minimum') runs = block['runs'] actual_ids = [run['job_id'] for run in runs] if len(runs) != len(jobs) or len(set(actual_ids)) != len(jobs): errors.append(label + ': missing or duplicate run within block') if set(actual_ids) != set(identifiers): errors.append(label + ': block job set differs from plan') order = result['orders'][index] if sorted(order) != list(range(len(jobs))): errors.append(label + ': random order is not a complete permutation') elif actual_ids != [identifiers[k] for k in order]: errors.append(label + ': executed order differs from registered permutation') previous_run_end = block['started_ns'] for run in runs: runs_seen += 1 job = job_map[run['job_id']] row = run['measurement'] argv = job['argv'] requested_frames = round(float(argv[3]) * float(argv[5])) requested_burn_ns = max(float(argv[-1]), plan['burn_in_ms']) * 1_000_000 if row['frames'] != requested_frames or row['channels'] != 2: errors.append(label + ': frame/channel count differs from CLI') if row['case_id'] != job['case_id']: errors.append(label + ': returned case ID differs from plan') if row['burn_in_ns'] < requested_burn_ns or row['burn_in_cycles'] < 1: errors.append(label + ': burn-in below requested minimum') minimum_burn = row['burn_in_ns'] if minimum_burn is None else min(minimum_burn, row['burn_in_ns']) if not all(math.isfinite(row[k]) for k in ('elapsed_ns', 'ns_per_frame', 'burn_in_ns', 'peak', 'rms')): errors.append(label + ': nonfinite measurement field') if row['elapsed_ns'] <= 0 or not math.isclose(row['ns_per_frame'], row['elapsed_ns']/row['frames'], rel_tol=1e-6): errors.append(label + ': elapsed/frame arithmetic mismatch') if not 0 <= row['rms'] <= row['peak']: errors.append(label + ': invalid peak/RMS relation') if row['state_bytes'] < 0 or row['latency_samples'] < 0: errors.append(label + ': negative state or latency') if not previous_run_end <= run['started_ns'] <= run['finished_ns'] <= block['finished_ns']: errors.append(label + ': non-monotonic run timestamps') previous_run_end = run['finished_ns'] fingerprint = {key: row[key] for key in FINGERPRINT_FIELDS} fingerprint.update({key: row[key] for key in OPTIONAL_FIELDS if key in row}) if run['job_id'] in fingerprints and fingerprints[run['job_id']] != fingerprint: errors.append(label + ': output/control fingerprint changed between blocks: ' + run['job_id']) fingerprints[run['job_id']] = fingerprint previous_block = block comparisons = result['analysis']['comparisons'] if [(p['reference'],p['candidate']) for p in comparisons] != [(p['reference'],p['candidate']) for p in plan['comparisons']]: errors.append(label + ': analyzed comparison list differs from plan') for pair in comparisons: if pair['blocks'] != len(blocks) or pair['df'] != len(blocks)-1: errors.append(label + ': analyzed pair count/degrees of freedom differ') return {'jobs':len(jobs), 'blocks':len(blocks), 'trials':runs_seen, 'seed':plan['seed'], 'cpu':plan['cpu'], 'minimum_reported_burn_in_ns':minimum_burn, 'minimum_recorded_block_gap_ns':minimum_gap, 'fingerprints':fingerprints} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--root', type=Path, default=ROOT) parser.add_argument('--output', type=Path) parser.add_argument('--family', action='append', help='Audit a named completed family; repeat to select several.') parser.add_argument('--metadata-only', action='store_true', help='Compare recorded binary hashes without requiring archived executables locally.') args = parser.parse_args() root = args.root.resolve() registration_path = root/'common/confirmation/registration.json' registration = json.loads(registration_path.read_text()) selected = args.family or list(registration['families']) if len(set(selected)) != len(selected) or not set(selected) <= set(registration['families']): parser.error('Unknown or duplicate family') output = args.output or root/'common/confirmation/audit.json' records = [] binary_cache = {} for family in selected: errors = [] record = {'family':family, 'errors':errors} try: entry = registration['families'][family] original_path = root/entry['original_result'] result_path = root/f'common/confirmation/{family}-performance.json' plan_path = root/entry['plan'] if not result_path.exists(): raise ValueError('confirmation result is missing') original = json.loads(original_path.read_text()) result = json.loads(result_path.read_text()) plan = json.loads(plan_path.read_text()) if sha(original_path) != entry['original_result_sha256']: errors.append('original result hash differs from registration') if sha(plan_path) != entry['plan_sha256'] or result['plan_sha256'] != entry['plan_sha256']: errors.append('confirmation plan hash differs from registration') if result['plan'] != plan: errors.append('embedded plan differs from registered file') if plan['blocks'] != registration['blocks'] or plan['cooldown_seconds'] != registration['cooldown_seconds'] or plan['burn_in_ms'] != registration['burn_in_ms']: errors.append('global confirmation conditions differ from registration') original_jobs = {j['id']:j for j in original['plan']['jobs']} if set(original_jobs) != {j['id'] for j in plan['jobs']}: errors.append('job selection differs from exploration') for job in plan['jobs']: previous = original_jobs[job['id']] if job['case_id'] != previous['case_id'] or job['argv'][:-1] != previous['argv'][:-1]: errors.append('case/CLI changed beyond burn-in: '+job['id']) if float(job['argv'][-1]) != registration['burn_in_ms']: errors.append('CLI burn-in differs from registration') if plan['comparisons'] != original['plan']['comparisons']: errors.append('comparison selection differs from exploration') if result['binary_sha256'] != entry['binary_sha256'] or original['binary_sha256'] != entry['binary_sha256']: errors.append('recorded binary hashes differ from fixed registration') if not args.metadata_only: for relative, expected in entry['binary_sha256'].items(): if relative not in binary_cache: binary_cache[relative] = sha(root/relative) if binary_cache[relative] != expected: errors.append('current executable hash differs: '+relative) if result['runner_sha256'] != original['runner_sha256'] or result['statistics_sha256'] != original['statistics_sha256']: errors.append('runner/statistics source changed between measurements') for key, relative in [('runner_sha256', 'common/run_blocks.py'), ('statistics_sha256', 'common/standard_statistics.py')]: if sha(root/relative) != result[key]: errors.append('archived measurement source differs: '+relative) first = audit_structure(original, errors, 'exploration') second = audit_structure(result, errors, 'confirmation') if first['fingerprints'] != second['fingerprints']: errors.append('exploration/confirmation audio/control fingerprint differs') record['exploration'] = {k:v for k,v in first.items() if k!='fingerprints'} record['confirmation'] = {k:v for k,v in second.items() if k!='fingerprints'} record['original_result_sha256'] = sha(original_path) record['confirmation_result_sha256'] = sha(result_path) record['fingerprints_equal'] = first['fingerprints'] == second['fingerprints'] except Exception as error: errors.append(type(error).__name__+': '+str(error).replace(str(root),'.')) record['passed'] = not errors records.append(record) report = {'checked_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'registration_sha256':sha(registration_path), 'scope':'all registered families' if args.family is None else 'explicit completed-family subset', 'registered_families':len(registration['families']), 'selected_families':selected, 'current_executable_verification_enabled':not args.metadata_only, 'distinct_current_executables_hashed':len(binary_cache), 'statistical_tests_recomputed':False, 'all_selected_passed':all(row['passed'] for row in records), 'families':records} output.parent.mkdir(parents=True,exist_ok=True) output.write_text(json.dumps(report,indent=2,allow_nan=False)+'\n') print(json.dumps({'families':len(records),'all_selected_passed':report['all_selected_passed'],'failed':[r['family']for r in records if not r['passed']]})) return 0 if report['all_selected_passed'] else 1 if __name__=='__main__': raise SystemExit(main())