Official StateBench generator and evaluator, commit 5b333b8d4266. Independent interpreter cross-checks every replay. No language model is called.
001 """Replay official StateBench traces; no language model is called.
002
003 This is a deterministic deletion audit, not a reproduction of model evaluations.
004 The upstream generator/evaluator are imported unchanged at the pinned commit.
005 """
006 from __future__ import annotations
007
008 import csv
009 import hashlib
010 import html
011 import json
012 import platform
013 import random
014 import subprocess
015 import sys
016 from collections import Counter, defaultdict
017 from datetime import datetime, timezone
018 from pathlib import Path
019
020 from state_bench.config import load_formatter
021 from state_bench.generation import derive_variants, generate_periodic_source
022 from state_bench.model import AssignConstant, AssignIncrement, AssignVariable, QueryMode
023 from state_bench.rendering import render_instance
024 from state_bench.semantics import evaluate
025
026 ROOT = Path(__file__).resolve().parent
027 REPO = ROOT / "vendor" / "state-bench"
028 RESULTS = ROOT / "results"
029 PIN = "5b333b8d426638d8825f40efe8a508c20eb85730"
030 SEEDS = tuple(range(32))
031 LENGTHS = (8, 32, 128, 512)
032 NUM_VARIABLES = 5
033 NUM_VALUES = 32
034
035
036 def json_write(path, data):
037 path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
038
039
040 def own_replay(operations, *, omit=None):
041 """Small independent interpreter, including uninitialized prefix snapshots."""
042 state = [None] * (NUM_VARIABLES + 1)
043 snapshots = []
044 for index, op in enumerate(operations):
045 if index != omit:
046 if isinstance(op, AssignConstant):
047 state[op.target] = op.value
048 elif isinstance(op, AssignVariable):
049 assert state[op.source] is not None
050 state[op.target] = state[op.source] # eager value copy
051 elif isinstance(op, AssignIncrement):
052 assert state[op.target] is not None
053 state[op.target] = (state[op.target] + op.delta) % NUM_VALUES
054 else:
055 raise TypeError(op)
056 snapshots.append(tuple(state))
057 return snapshots
058
059
060 def official_replay(instance, operations):
061 return evaluate(num_variables=instance.num_registers,
062 num_values=instance.num_values, operations=operations)
063
064
065 def op_text(op):
066 name = lambda n: "TMP" if n == NUM_VARIABLES else chr(97+n)
067 if isinstance(op, AssignConstant):
068 return f"{name(op.target)} <- {op.value};"
069 if isinstance(op, AssignVariable):
070 return f"{name(op.target)} <- {name(op.source)};"
071 return f"{name(op.target)} <- ({name(op.target)} + {op.delta}) mod {NUM_VALUES};"
072
073
074 def eager_copy_check():
075 # 2 ordinary registers, no TMP: changing b later must not change a.
076 ops = (AssignConstant(0, 2), AssignConstant(1, 5),
077 AssignVariable(0, 1), AssignConstant(1, 9))
078 result = evaluate(num_variables=2, num_values=32, operations=ops)
079 assert result == (5, 9)
080 return {"operations": [op.to_record() for op in ops],
081 "expected": [5, 9], "actual": list(result),
082 "explanation": "a copies b's value 5 immediately; b's later 9 does not change a."}
083
084
085 def main():
086 RESULTS.mkdir(exist_ok=True)
087 commit = subprocess.check_output(["git", "-C", str(REPO), "rev-parse", "HEAD"], text=True).strip()
088 assert commit == PIN
089 formatter = load_formatter(REPO / "configs/formats/integer-code-modular.yaml")
090 eager = eager_copy_check()
091 summary = defaultdict(Counter)
092 invariants = Counter()
093 fixture = None
094 csv_path = RESULTS / "omission_replays.csv"
095 sources_path = RESULTS / "canonical_family_sets.jsonl"
096 with csv_path.open("w", newline="", encoding="utf-8") as cf, sources_path.open("w", encoding="utf-8") as jf:
097 fields = ["seed", "extra_assignments", "family", "source_instance_id",
098 "operation_index_0based", "update_index_1based", "omitted_op_type",
099 "omitted_target", "ordinary_mismatches", "all_register_mismatches"]
100 writer = csv.DictWriter(cf, fieldnames=fields)
101 writer.writeheader()
102 for length in LENGTHS:
103 for seed in SEEDS:
104 kwargs = dict(num_variables=NUM_VARIABLES, num_values=NUM_VALUES,
105 min_extra_assignments=length, max_extra_assignments=length,
106 min_aperiodic_constants=1, max_aperiodic_constants=1,
107 distinct_initial_values=True, query_mode=QueryMode.ALL)
108 source = generate_periodic_source(**kwargs, rng=random.Random(seed))
109 repeated = generate_periodic_source(**kwargs, rng=random.Random(seed))
110 assert source == repeated
111 invariants["deterministic_source_regeneration"] += 1
112 variants = derive_variants(source)
113 snapshots = {v.complexity.value: own_replay(v.operations) for v in variants}
114 assert snapshots["periodic"] == snapshots["aperiodic"] == snapshots["solvable"]
115 invariants["three_family_stepwise_state_equality"] += len(source.operations)
116 initial_count = source.num_initial_assignments
117 assert initial_count == NUM_VARIABLES + 1
118 assert len(source.operations) == initial_count + length
119 assert all(isinstance(op, AssignConstant) for op in source.operations[:initial_count])
120 init_state = snapshots["periodic"][initial_count - 1]
121 renders = []
122 for instance in variants:
123 family = instance.complexity.value
124 assert len(instance.operations) == len(source.operations)
125 assert tuple(op.target for op in instance.operations) == tuple(op.target for op in source.operations)
126 assert instance.query == source.query
127 assert instance.temporary_variable == source.temporary_variable
128 assert tuple(snapshots[family][-1]) == instance.final_state
129 assert official_replay(instance, instance.operations) == instance.final_state
130 invariants["official_and_independent_final_evaluator_agreement"] += 1
131 if family == "r-trivial":
132 assert all(state == init_state for state in snapshots[family][initial_count - 1:])
133 invariants["r_trivial_first_value_invariant"] += 1
134 rendered = render_instance(instance, formatter, rng=random.Random(991))
135 renders.append(rendered)
136 key = (length, family)
137 summary[key]["source_count"] += 1
138 for index in range(initial_count, len(instance.operations)):
139 omitted = instance.operations[:index] + instance.operations[index + 1:]
140 observed = official_replay(instance, omitted)
141 independently_observed = own_replay(instance.operations, omit=index)[-1]
142 assert observed == independently_observed
143 mismatches = sum(a != b for a,b in zip(observed[:NUM_VARIABLES], instance.final_state[:NUM_VARIABLES]))
144 all_mismatches = sum(a != b for a,b in zip(observed, instance.final_state))
145 row = dict(seed=seed, extra_assignments=length, family=family,
146 source_instance_id=source.instance_id, operation_index_0based=index,
147 update_index_1based=index-initial_count+1,
148 omitted_op_type=instance.operations[index].to_record()["op"],
149 omitted_target=instance.operations[index].target,
150 ordinary_mismatches=mismatches,
151 all_register_mismatches=all_mismatches)
152 writer.writerow(row)
153 summary[key]["omission_replays"] += 1
154 summary[key]["replays_with_ordinary_error"] += mismatches > 0
155 summary[key]["ordinary_mismatch_sum"] += mismatches
156 summary[key]["replays_with_any_register_error"] += all_mismatches > 0
157 summary[key]["all_register_mismatch_sum"] += all_mismatches
158 summary[key][f"ordinary_mismatch_{mismatches}"] += 1
159 invariants["omission_official_independent_agreement"] += 1
160 assert len({r.surface_variables for r in renders}) == 1
161 assert len({r.surface_values for r in renders}) == 1
162 invariants["four_family_structure_and_surface_alignment"] += 1
163 jf.write(json.dumps({"seed": seed, "extra_assignments": length,
164 "source_instance_id": source.instance_id,
165 "variants": [v.to_record() for v in variants]}) + "\n")
166 if length == 8 and seed == 0:
167 fixture = {"seed": seed, "extra_assignments": length,
168 "scope": "Actual upstream generator output; no language model.",
169 "source_instance_id": source.instance_id,
170 "num_variables": NUM_VARIABLES, "num_values": NUM_VALUES,
171 "temporary_variable": source.temporary_variable,
172 "scored_registers": list(range(NUM_VARIABLES)),
173 "variants": [{**v.to_record(), "readable_operations": [op_text(op) for op in v.operations],
174 "state_after_each_assignment": snapshots[v.complexity.value],
175 "official_rendering": r.to_record()}
176 for v,r in zip(variants,renders)]}
177 print(f"length={length} seed={seed} checked", flush=True)
178 rows = []
179 for (length, family), counts in summary.items():
180 rows.append({"extra_assignments": length, "family": family, **counts,
181 "ordinary_error_survival_rate": counts["replays_with_ordinary_error"]/counts["omission_replays"],
182 "ordinary_error_survival_percent": 100*counts["replays_with_ordinary_error"]/counts["omission_replays"],
183 "mean_ordinary_mismatches": counts["ordinary_mismatch_sum"]/counts["omission_replays"],
184 "all_register_error_survival_rate": counts["replays_with_any_register_error"]/counts["omission_replays"]})
185 json_write(RESULTS / "summary.json", rows)
186 keys = sorted({k for row in rows for k in row})
187 with (RESULTS / "summary.csv").open("w", newline="", encoding="utf-8") as f:
188 w=csv.DictWriter(f,fieldnames=keys); w.writeheader(); w.writerows(rows)
189 json_write(RESULTS / "fixture_seed0_updates8.json", fixture)
190 json_write(RESULTS / "eager_copy_check.json", eager)
191 provenance = {"upstream_repository": "https://github.com/allenai/state-bench",
192 "commit": commit, "license": "Apache-2.0",
193 "run_utc": datetime.now(timezone.utc).isoformat(),
194 "python": sys.version, "platform": platform.platform(),
195 "seeds": SEEDS, "extra_primitive_assignments": LENGTHS,
196 "ordinary_variables": NUM_VARIABLES, "scratch_registers": 1,
197 "num_values": NUM_VALUES, "distinct_initial_values": True,
198 "aperiodic_constants_min_max": [1,1], "filler_proportion": 0,
199 "omission_policy": "Every post-initialization primitive assignment is deleted once in a separate replay. This includes TMP loads and individual swap assignments, not whole swaps.",
200 "score": "Any mismatch among the five ordinary final registers; TMP excluded. All-register supplementary results also saved.",
201 "not_model_evaluation": True, "model_calls": 0,
202 "scope": "Small deterministic audit of unmodified official generator/evaluator; settings differ from paper model evaluations.",
203 "invariant_assertions": dict(invariants)}
204 json_write(RESULTS / "provenance.json", provenance)
205 source = (ROOT / "run_audit.py").read_text(encoding="utf-8")
206 numbered = "\n".join(f"{i:03} {line}" for i,line in enumerate(source.splitlines(),1))
207 (RESULTS / "source-code.html").write_text("<!doctype html><html lang='en'><meta charset='utf-8'><title>StateBench deletion audit source</title><style>body{background:#101720;color:#d9e7ef;font:16px/1.55 Consolas,monospace;padding:32px}h1{font:700 28px system-ui;color:#8ae9d0}p{font:16px system-ui;max-width:1000px}pre{white-space:pre-wrap}code{tab-size:4}</style><h1>Replay one missing assignment</h1><p>Official StateBench generator and evaluator, commit "+PIN[:12]+". Independent interpreter cross-checks every replay. No language model is called.</p><pre><code>"+html.escape(numbered)+"</code></pre></html>",encoding="utf-8")
208 fixture_lines=[]
209 by_family={v["complexity"]:v for v in fixture["variants"]}
210 for index in range(NUM_VARIABLES+1+8):
211 p=by_family["periodic"]
212 fixture_lines.append(f"{index+1:02} {p['readable_operations'][index]:18} "+str(p["state_after_each_assignment"][index]))
213 fixture_html="<!doctype html><html lang='en'><meta charset='utf-8'><title>Actual StateBench trace</title><style>body{margin:0;background:#101720;color:#d9e7ef;font:20px/1.7 Consolas,monospace;padding:48px}h1{font:700 34px system-ui;color:#8ae9d0}p{font:20px/1.5 system-ui;max-width:1000px}small{color:#9cb0c1}pre{padding:28px;background:#182432;border:1px solid #375466;border-radius:16px;display:inline-block}</style><h1>One actual generated trace</h1><p>Seed 0 · five ordinary registers + TMP · 8 update assignments · 32 values</p><small>Columns: assignment / state [a, b, c, d, e, TMP] immediately afterward</small><br><pre>"+html.escape("\n".join(fixture_lines))+"</pre><p>Copies happen immediately. A later change to the source register does not change a value already copied.</p><small>Upstream commit "+PIN[:12]+" · Apache-2.0 · deterministic replay · no model calls</small></html>"
214 (RESULTS / "actual-trace.html").write_text(fixture_html,encoding="utf-8")
215 hashes={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in RESULTS.iterdir() if p.is_file() and p.name!="sha256.json"}
216 json_write(RESULTS / "sha256.json", hashes)
217 print(json.dumps({"summary":rows,"assertions":dict(invariants)},indent=2))
218
219
220 if __name__ == "__main__":
221 main()