#!/usr/bin/env python3 """Emit each task's agent-visible evaluator copy from this package. $SERVERBENCH_PY -m final_validate.package_grader --write $SERVERBENCH_PY -m final_validate.package_grader --check # drift only A task's `tests/final_verdict/` is a byte-for-byte copy of the measurement half of this package -- the AgentX replay, the protocol probes, the scoring -- so the agent develops against the code that measures it. The copy is self-contained (the repository is not mounted into an agent container) and carries only this task's own module. Never copied: how output correctness is judged (`private_correctness`, `agentx_reference`), device leasing and deployment, the Modal backend, and the operator tools. `check` refuses a copy that carries any of `NEVER_PACKAGED`. """ from __future__ import annotations import argparse import shutil from pathlib import Path PACKAGE = Path(__file__).resolve().parent REPO = PACKAGE.parents[2] #: A task directory contains both its contract and runnable Harbor package. HARBOR_TASKS = REPO / "serving_tasks" #: Copied into every task's evaluator. A module imported by the copy but #: missing here is an ImportError inside the container. SHARED_MODULES = ( "arms.py", "candidate_bundle.py", "native_tokens.py", "agentx_pin.py", "objective.py", "token_prices.py", "service_levels.py", "stream_timing.py", "spine.py", "task_spec.py", "scenario.py", "replay_verify.py", "warmup_watch.py", ) #: How output correctness is judged. Checked absent from every copy. NEVER_PACKAGED = ("agentx_reference.py", "private_correctness.py", "pool") GRADER_DIR = "tests/final_verdict" #: Build noise inside the copy, which is not a stray file. EXCLUDED = ("__pycache__", ".pytest_cache") TEST_SH = """#!/usr/bin/env bash # Harbor verifier entry point for {task}. GENERATED by # serverbench/harness/final_validate/package_grader.py; do not edit. # # This task's verdict is an 1800-second AgentX replay followed by a private # correctness stage, run by the host (`final_validate/scenario.py`) after the # evolve ends. It cannot run inside a Harbor verifier container, so this entry # point refuses and writes no reward file: Harbor records an exception, never # a score. echo "{task}: the verdict runs on the host via final_validate/scenario.py," \\ "not in the Harbor verifier" >&2 exit 1 """ REPLAY_VERIFY_SH = """#!/usr/bin/env bash # Generated convenience entry point for the public AgentX replay preflight. set -euo pipefail HERE="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)" cd "$HERE" exec python3 -m final_verdict.replay_verify "{task}" "$@" """ INIT = ('"""Generated copy of the final verdict\'s measurement code, ' 'byte-for-byte from serverbench/harness/final_validate/."""\n') def _module_for(task_name: str) -> Path: """This task's own module: the lowercase full task id, never a prefix.""" module = PACKAGE / "tasks" / f"{task_name.lower()}.py" if not module.is_file(): raise FileNotFoundError( f"{task_name} has no module under {PACKAGE / 'tasks'}") return module def plan(task_dir: Path) -> dict[str, Path]: """`destination relative path -> source file`, for one task.""" out: dict[str, Path] = {} for name in SHARED_MODULES: source = PACKAGE / name if not source.is_file(): raise FileNotFoundError(f"{source} is in SHARED_MODULES but absent") out[f"{GRADER_DIR}/{name}"] = source out[f"{GRADER_DIR}/tasks/__init__.py"] = PACKAGE / "tasks" / "__init__.py" # Shared helpers under `tasks/` start with an underscore. for helper in sorted((PACKAGE / "tasks").glob("_*.py")): out[f"{GRADER_DIR}/tasks/{helper.name}"] = helper module = _module_for(task_dir.name) out[f"{GRADER_DIR}/tasks/{module.name}"] = module return out def _generated(task_name: str) -> dict[str, tuple[str, int]]: """Files packaging writes rather than copies: path -> (text, mode).""" return { f"{GRADER_DIR}/__init__.py": (INIT, 0o644), "tests/test.sh": (TEST_SH.format(task=task_name), 0o755), "tests/replay-verify": (REPLAY_VERIFY_SH.format(task=task_name), 0o755), } def write(task_dir: Path) -> dict: files = plan(task_dir) root = task_dir / GRADER_DIR if root.exists(): shutil.rmtree(root) for name, source in files.items(): target = task_dir / name target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, target) for name, (text, mode) in _generated(task_dir.name).items(): target = task_dir / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text(text, "utf-8") target.chmod(mode) return {"files": sorted(files)} def check(task_dir: Path) -> list[str]: """Drift between a task's copy and this package. Empty means in step.""" root = task_dir / GRADER_DIR if not root.is_dir(): return [f"{task_dir.name}: no evaluator copy under {GRADER_DIR}"] problems: list[str] = [] files = plan(task_dir) generated = _generated(task_dir.name) for name, source in files.items(): target = task_dir / name if not target.is_file(): problems.append(f"{task_dir.name}: {name} is missing") elif target.read_bytes() != source.read_bytes(): problems.append( f"{task_dir.name}: {name} differs from its source in " f"serverbench/harness/final_validate/; regenerate the package") for name, (text, mode) in generated.items(): target = task_dir / name if not target.is_file() or target.read_text("utf-8") != text: problems.append(f"{task_dir.name}: {name} differs from the " f"generated file; regenerate the package") elif mode & 0o111 and not target.stat().st_mode & 0o111: problems.append(f"{task_dir.name}: {name} is not executable") expected = set(files) | set(generated) stray = sorted( str(p.relative_to(task_dir)) for p in root.rglob("*") if p.is_file() and not any(part in EXCLUDED for part in p.parts) and str(p.relative_to(task_dir)) not in expected) if stray: problems.append(f"{task_dir.name}: the evaluator copy carries files " f"packaging does not emit: {stray[:4]}") for name in NEVER_PACKAGED: if (root / name).exists(): problems.append( f"{task_dir.name}: {GRADER_DIR}/{name} is correctness code or " f"data; the agent-visible copy must not carry it") return problems def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) mode = ap.add_mutually_exclusive_group(required=True) mode.add_argument("--write", action="store_true", help="regenerate every task's evaluator copy") mode.add_argument("--check", action="store_true", help="report drift and change nothing") ap.add_argument("--task", default="", help="one task directory name; defaults to all of them") args = ap.parse_args(argv) tasks = ([HARBOR_TASKS / args.task] if args.task else sorted(p for p in HARBOR_TASKS.iterdir() # spine.is_task_dir if p.is_dir() and not p.name.startswith(("_", ".")) and (p / "contract.yaml").is_file())) if not tasks: raise SystemExit(f"no packaged tasks under {HARBOR_TASKS}") if args.check: problems = [line for task in tasks for line in check(task)] for line in problems: print(f" {line}") broken = {line.split(":")[0].strip() for line in problems} print(f"\n{len(tasks) - len(broken)}/{len(tasks)} evaluator copies in " f"step with serverbench/harness/final_validate/") return 1 if problems else 0 for task in tasks: try: body = write(task) except FileNotFoundError as exc: print(f" {task.name}: SKIPPED -- {exc}") continue print(f" {task.name}: {len(body['files'])} file(s)") print(f"\nwrote {len(tasks)} evaluator copies from serverbench/harness/final_validate/") return 0 if __name__ == "__main__": raise SystemExit(main())