#!/usr/bin/env python3 # Copyright (C) 2026 Yusuke Oda # SPDX-License-Identifier: AGPL-3.0-only """Predeclared multi-method randomized-block effects measurements.""" import argparse import datetime import fcntl import hashlib import json import math import os from pathlib import Path import random import subprocess import sys import time import standard_statistics as statistics ROOT = Path(__file__).resolve().parent.parent def utc(): return datetime.datetime.now(datetime.timezone.utc).isoformat() def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def dump(path, data): temporary = path.with_suffix(path.suffix + '.tmp') temporary.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') temporary.replace(path) def validate(row, job, plan): for name in ('elapsed_ns','ns_per_frame','burn_in_ns','peak','rms'): if not math.isfinite(row[name]): raise ValueError('nonfinite ' + name) if (row['channels'] != 2 or row['elapsed_ns'] <= 0 or row['frames'] <= 0 or row['burn_in_ns'] < plan['burn_in_ms'] * 1_000_000 or row['burn_in_cycles'] < 1 or not 0 <= row['rms'] <= row['peak'] or row['state_bytes'] < 0 or row['latency_samples'] < 0 or row['case_id'] != job['case_id'] or not isinstance(row['checksum'], str) or not math.isclose(row['ns_per_frame'], row['elapsed_ns']/row['frames'], rel_tol=1e-6)): raise ValueError('invalid row: ' + str(row)) def analyze(result): jobs = result['plan']['jobs'] samples = {job['id']: [] for job in jobs} for block in result['blocks']: if {r['job_id'] for r in block['runs']} != set(samples): raise ValueError('incomplete block') for run in block['runs']: samples[run['job_id']].append(run['measurement']['ns_per_frame']) comparisons=[] for pair in result['plan']['comparisons']: item=statistics.paired_statistics(samples[pair['reference']], samples[pair['candidate']]) item.update(pair) item['mean_ratio']=item['after']['mean']/item['before']['mean'] comparisons.append(item) for item,p in zip(comparisons, statistics.holm([item['p'] for item in comparisons])): item['p_holm']=p return {'unit':'ns/stereo-frame','method':'paired two-sided t, Holm across predeclared comparisons', 'summaries':{key:statistics.summary(value) for key,value in samples.items()}, 'comparisons':comparisons} def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('plan',type=Path) p.add_argument('output',type=Path) p.add_argument('--analyze-only',action='store_true') args=p.parse_args() if args.analyze_only: result=json.loads(args.output.read_text()) result['analysis']=analyze(result) dump(args.output,result) return if args.output.exists(): raise ValueError('Output already exists; never overwrite a measurement') plan=json.loads(args.plan.read_text()) if plan['blocks']<10 or plan['cooldown_seconds']<3 or plan['burn_in_ms']<10: raise ValueError('insufficient repetitions/warmup/cooldown') jobs=plan['jobs'] identifiers={j['id'] for j in jobs} if len(identifiers)!=len(jobs): raise ValueError('duplicate job id') for pair in plan['comparisons']: if not {pair['reference'],pair['candidate']}<=identifiers: raise ValueError('unknown comparison job') binaries={} for job in jobs: binary=(ROOT/job['argv'][0]).resolve() if not binary.is_relative_to(ROOT): raise ValueError('binary outside reproducible study tree') binaries[job['argv'][0]]=sha(binary) rng=random.Random(plan['seed']) orders=[] for _ in range(plan['blocks']): order=list(range(len(jobs))) rng.shuffle(order) orders.append(order) result={'schema':1,'started_utc':utc(),'plan_sha256':sha(args.plan),'plan':plan, 'environment':json.loads((ROOT/'common/environment.json').read_text()), 'runner_sha256':sha(__file__),'statistics_sha256':sha(Path(__file__).with_name('standard_statistics.py')), 'binary_sha256':binaries,'orders':orders,'status':'running','blocks':[]} args.output.parent.mkdir(parents=True,exist_ok=True) dump(args.output,result) fingerprints={} origin=time.perf_counter_ns() lock=(ROOT/'measurement.lock').open('w') fcntl.flock(lock, fcntl.LOCK_EX) (ROOT/'measurement-active.json').write_text(json.dumps({'started_utc':utc(),'plan':args.plan.name,'pid':os.getpid()})) try: for number,order in enumerate(orders): if number: time.sleep(plan['cooldown_seconds']) block={'number':number+1,'started_utc':utc(),'started_ns':time.perf_counter_ns()-origin,'runs':[]} print(f"{args.plan.stem}: block {number+1}/{plan['blocks']} ({len(jobs)} runs)",flush=True) for index in order: job=jobs[index] argv=[str(ROOT/job['argv'][0]),*job['argv'][1:]] started=time.perf_counter_ns()-origin stamp=utc() raw=subprocess.check_output(['taskset','-c',str(plan['cpu']),*argv],cwd=ROOT,text=True) finished=time.perf_counter_ns()-origin row=json.loads(raw) validate(row,job,plan) check=(row['checksum'],row['peak'],row['rms']) if job['id'] in fingerprints and check!=fingerprints[job['id']]: raise ValueError('non-reproducible output: '+job['id']) fingerprints[job['id']]=check block['runs'].append({'job_id':job['id'],'started_utc':stamp,'started_ns':started,'finished_ns':finished,'measurement':row}) block['finished_ns']=time.perf_counter_ns()-origin result['blocks'].append(block) dump(args.output,result) result.update(status='complete',finished_utc=utc(),analysis=analyze(result)) dump(args.output,result) except BaseException as error: result.update(status='failed',finished_utc=utc(),error=repr(error)) dump(args.output,result) raise finally: (ROOT/'measurement-active.json').unlink(missing_ok=True) fcntl.flock(lock,fcntl.LOCK_UN) lock.close() if __name__=='__main__': main()