"""Production lifecycle: build, develop, checkpoint, preserve, then validate.""" from __future__ import annotations import argparse import atexit import contextlib import dataclasses import datetime import hashlib import json import os import shutil import signal import subprocess import sys import threading import time import traceback from pathlib import Path from . import agent as agent_mod from . import backend, boundary_checks from . import billing as billing_mod from . import instructions as instructions_mod from . import publish as publish_mod from . import record as ids from . import report as report_mod from . import workspace as ws_mod from .boundary import CaptureStore, VersionRegistry from .publish import CASConflict, Publisher from .record import ( CandidateState, Journal, RunState, candidate_state, candidates, move_candidate, move_run, run_state, ) # Nothing in a run measures the candidate: the agent measures itself in-Harbor # with the generated copy of the final verdict, and the record comes from the # final verdict run OUTSIDE Harbor afterwards. from .utils import describe from .verdict import FailureClass from .workspace import Ownership, container_is_up, device_uuid, foreign_processes #: A RUNAWAY CEILING, not a continuation budget. #: #: A launch that WORKED is resumed for as long as the deadline allows, so the #: budget is `--hours` rather than a launch count; a flat count would let a #: loop of short idle turns end a productive run early. What is bounded is #: CONSECUTIVE IDLE launches, below. This number exists only so a pathological #: loop cannot hold a card until the deadline. MAX_LAUNCHES = 400 #: How many launches in a row may end having done nothing -- see #: `agent.was_idle` for what that means -- before the #: run stops asking. The first idle launch is a turn that ended early, and the #: next launch is given the task's `## Idle restart` section, which states #: what this environment does not do. Four of those in a row, each after a #: backoff, is an agent that is not going to start. MAX_IDLE_LAUNCHES = 4 #: The wait before an idle relaunch: this many seconds per launch of the #: current idle streak, capped, so an idle loop cannot burn launches in #: seconds. The wait is NOT added back to the deadline: the agent had budget #: and did not use it, and `--hours` still bounds its wall time. IDLE_BACKOFF_S = 30.0 IDLE_BACKOFF_CAP_S = 300.0 #: Below this much budget a cold restart is not worth the model load and #: the workspace rebuild; the run stops instead. COLD_RESTART_FLOOR_S = 15 * 60 #: Fallback ceiling for the closing final validation, used when the task's own #: cost cannot be derived (`final_validation_timeout`). Bounded, because the #: card is still held and a run that never ends never gives it back. FINAL_VALIDATION_TIMEOUT_S = 5400.0 #: What one arm costs BESIDES its benchmark: loading the weights onto the #: cards, rebuilding a scenario's corpus from KV block hashes, warmup, and the #: teardown that follows. Generous on purpose: the cost of overestimating is a #: timeout that never fires. ARM_STARTUP_ALLOWANCE_S = 900.0 #: The verdict does not end with the benchmark. Its private correctness stage #: serves the plain checkpoint as the reference and runs the answer, EOS-stop #: and context-fidelity probes against both. Sized for a large task whose #: warmup runs long prefills, and for the reference engine's own 1800 s #: readiness limit (`deploy.BASELINE_READY_TIMEOUT_S`). CORRECTNESS_STAGE_ALLOWANCE_S = 6300.0 def _agent_benchmarks(workspace) -> int: """How many non-preflight aiperf runs the agent completed. Counts finished ones only -- an export directory appears when a benchmark STARTS and its summary is written at the end -- and ignores the harness's `.review/` snapshots, which are copies rather than runs. The lightweight replay verifier also writes an aiperf aggregate, but it is deliberately not a performance measurement; counting it here would erase the distinction its output and the task brief make explicit. """ if not workspace: return 0 root = Path(workspace) if not root.is_dir(): return 0 found = 0 for path in root.rglob("profile_export_aiperf.json"): try: parts = path.relative_to(root).parts except ValueError: continue if any(part.startswith(".") for part in parts[:-1]): continue if "lightweight-agentx" in parts: continue found += 1 return found def _agent_replay_verifications(workspace) -> int: """How many lightweight AgentX replay preflights wrote a report.""" if not workspace: return 0 root = Path(workspace) if not root.is_dir(): return 0 found = 0 for path in root.rglob("replay-verification.json"): try: parts = path.relative_to(root).parts except ValueError: continue if any(part.startswith(".") for part in parts[:-1]): continue found += 1 return found def _agent_correctness_runs(workspace) -> int: """How many CORRECTNESS checks the agent completed in its own workspace. Counted apart from performance runs because they are different acts and the difference is what an operator needs: an agent that checked correctness but never timed a replay has done half the job, not none. `--check-protocol-context` runs the endpoint probes and no timed replay, so it writes `stream-protocol-probe.json`, not a `profile_export_aiperf.json`. Each directory is counted once. The file does not prove that the full final correctness gate passed. """ if not workspace: return 0 root = Path(workspace) if not root.is_dir(): return 0 found = set() for path in root.rglob("stream-protocol-probe.json"): try: parts = path.relative_to(root).parts except ValueError: continue if any(part.startswith(".") for part in parts[:-1]): continue found.add(path.parent) return len(found) def _objective_detail(task_dir) -> dict: """The task's frozen objective inputs, from `contract.yaml`, or `{}`. The denominator and the token prices are ONE contract, so both come from one read; otherwise a trajectory point could price tokens with a table the task did not declare. The hourly denominator is DERIVED here -- the rate times the card count -- rather than read from a stored product that could disagree with its factors. The per-token prices are derived the same way. """ try: import yaml body = yaml.safe_load( (Path(task_dir) / "contract.yaml").read_text("utf-8")) or {} devices, rate = int(body["devices"]), float(body["usd_per_gpu_hour"]) prices = dict(body.get("token_prices") or {}) for kind in ("input", "output", "cached_input"): per_million = prices.get(f"usd_per_million_{kind}") if per_million is not None: prices[f"usd_per_{kind}_token"] = float(per_million) / 1e6 return {"objective": body.get("objective") or "dollars_per_dollar", "devices": devices, "devices_charged": devices, "usd_per_gpu_hour": rate, "usd_per_hour_total": rate * devices, "token_prices": prices} except Exception: # noqa: BLE001 return {} def _usd_per_hour(task_dir) -> float: """The task's frozen `objective_detail.usd_per_hour_total`, or 0.""" try: return float(_objective_detail(task_dir) .get("usd_per_hour_total") or 0.0) except (TypeError, ValueError): return 0.0 def verdict_seconds(task_dir) -> float: """How long the verdict's scored window is, from the task's own module. The development window and the scored one are the SAME length, so a pass the agent measures is a pass it gets. A summary that reports a request count should still say what the window was, because the request count is a property of the draw and the window is not. """ try: from .utils import ensure_harness_on_path ensure_harness_on_path() from final_validate import tasks as tasks_mod module = tasks_mod.python_module(Path(task_dir).name) return float(getattr(module, "VERDICT_DURATION_S", 0) or 0) except Exception: # noqa: BLE001 return 0.0 def _service_levels(): """`final_validate.service_levels`, imported lazily. Same reason as `_model_identity`: the gates belong to the verdict, and `tests/integrity/test_the_trajectory_never_reaches_the_verdict.py` forbids the traffic in the OTHER direction only. One module decides what a gate is, and the driver asks it rather than keeping a second copy that could drift. """ from .utils import ensure_harness_on_path ensure_harness_on_path() from final_validate import service_levels return service_levels def _version_order(version_id: str) -> tuple[int, str]: """`v10` after `v9`, which a string sort gets backwards.""" text = str(version_id) digits = text[1:] if text[:1] == "v" else text return (int(digits), text) if digits.isdigit() else (-1, text) def _model_identity(): """`final_validate.model_identity`, imported lazily. It lives on the verdict's side of the line `tests/integrity/test_the_trajectory_never_reaches_the_verdict.py` draws: BOTH the driver and the verdict must ask whether the weights on the card are the weights the task froze, and that test forbids the verdict importing anything from `lifecycle`. Lazily, because the harness core has to stay importable inside a bare image. """ from .utils import ensure_harness_on_path ensure_harness_on_path() from final_validate import model_identity return model_identity def declared_model(task_dir) -> tuple[str, str]: """The task's frozen `(model_id, revision)`, or `("", "")`.""" return _model_identity().declared_for(task_dir) def final_validation_timeout(task_dir) -> float: """How long this task's verdict may take, derived rather than assumed. A scenario verdict replays its window after loading the model and rebuilding sessions from their KV block hashes, then runs the correctness stage; a fixed ceiling would kill a verdict that was measuring correctly. Derived from the task's own module, so a task with a different duration is right without editing this. """ try: from .utils import ensure_harness_on_path ensure_harness_on_path() from final_validate import tasks as tasks_mod task_id = Path(task_dir).name module = tasks_mod.python_module(task_id) duration = float(getattr(module, "VERDICT_DURATION_S", 0) or 0) if not duration: return FINAL_VALIDATION_TIMEOUT_S # ONE arm: `validate_evolve.py` passes `--arm candidate`, because the # baselines are frozen and measured once per task rather than at the # end of every run. Sized for two so a slow load or a retry has room. return 2 * (duration + ARM_STARTUP_ALLOWANCE_S) + CORRECTNESS_STAGE_ALLOWANCE_S except Exception: # noqa: BLE001 return FINAL_VALIDATION_TIMEOUT_S #: The task files a run is judged against. Their combined digest is the epoch. #: `contract.yaml` is the bars and the denominators; `reference.yaml` is the #: measured evidence a candidate is compared against. EPOCH_FILES = ("contract.yaml", "reference.yaml") def control_epoch(task_dir: Path) -> str: h = hashlib.sha256() for rel in EPOCH_FILES: path = Path(task_dir) / rel h.update(rel.encode()) h.update(path.read_bytes() if path.exists() else b"absent") return f"{Path(task_dir).name}-{h.hexdigest()[:12]}" def agent_slug(model: str, effort: str = "") -> str: """The agent that built a lineage, as a directory-name fragment. A lineage is one `(task, model, agent)` triple, and comparing agents on the same task is the benchmark's whole point, so the agent is part of the directory name rather than something a reader has to open `SUMMARY.json` to find. Effort is part of the identity because it changes what the agent is: opus at `high` and opus at `xhigh` are different budgets for the same weights, and a run comparing them would otherwise produce two names differing only in a timestamp. It is appended only when set. claude-opus-5, high -> opus-5-high z-ai/glm-5.3-flash -> glm-5.3-flash """ name = (model or "unknown").strip().lower() name = name.rsplit("/", 1)[-1] # drop an OpenRouter vendor prefix for prefix in ("claude-", "anthropic-"): if name.startswith(prefix): name = name[len(prefix):] slug = "".join(c if (c.isalnum() or c in "-.") else "-" for c in name) slug = "-".join(part for part in slug.split("-") if part) effort = "".join(c for c in (effort or "").strip().lower() if c.isalnum()) return f"{slug}-{effort}" if effort else (slug or "unknown") def persisted_agent_session(journal: Journal, provider: str) -> str: """Return the newest provider session recorded by a completed run.""" for event in reversed(journal.events()): kind = event.get("event") session_id = str(event.get("session_id") or "") if not session_id: continue if kind in {"agent_session", "agent_session_store_imported", "agent_session_imported"}: if event.get("provider") == provider: return session_id elif kind == "agent_launched" and event.get("agent_cli") == provider: return session_id return "" @dataclasses.dataclass class RunPlan: task_dir: Path repo: Path #: Every device this run holds, in the order the task's topology wants them. #: `gpu` is the FIRST of them and stays an int, because the bench service, #: the profiler and the headroom check each address one card by index and #: asking them to average four would make a per-device reading meaningless. gpus: tuple[int, ...] hours: float image: str model_dir: Path env_input: Path lineage_root: Path seed: Path | None = None agent_cli: str = "claude" agent_model: str = "claude-opus-5" agent_effort: str = "xhigh" #: A per-run KEY=VALUE file merged into the container's environment, for a #: run whose agent speaks to a different model backend. Empty means the #: host default. Recorded by PATH in `run_created`, never by content: which #: backend a run used is evidence, the credential is not. agent_env: Path | None = None credentials: Path | None = None profile: str = "opus" namespace: str = ids.PRODUCTION #: How long `finish` waits for extraction, deterministic checks and version #: materialisation after a checkpoint request. after_capture_timeout: float = 900.0 tasks_root: Path | None = None brief: str = "" lineage_id: str = "" model_id: str = "Qwen/Qwen3-4B" stop: str = "timebox" agents_dir: Path | None = None #: Claude Code project files layered into the workspace's `.claude/`. claude_project_dir: Path | None = None memory_dir: Path | None = None skill_dir: Path | None = None #: The second of the skills the operator's allowlist admits. Named #: separately rather than globbed, so adding one is a deliberate edit. ncu_skill_dir: Path | None = None #: The third. kernel_skill_dir: Path | None = None #: Complete pinned KernelWiki, delivered only to the instructed treatment. kernelwiki_skill_dir: Path | None = None #: Parent-owned L40/skill metadata layered over the clean KernelWiki #: submodule only in the agent's materialized copy. kernelwiki_overlay_dir: Path | None = None #: A completed Codex run whose provider thread supplies conversation state #: to this separately identified make-up run. Source code still comes from #: the lineage's published head unless `resume_workspace` is explicitly #: selected for recovery from an externally interrupted run. resume_run: Path | None = None #: Recover the source files that were present in `resume_run/workspace` at #: interruption time. Harness-owned files are regenerated; only the same #: candidate manifest that publication would capture crosses the boundary. resume_workspace: bool = False resume_session_id: str = "" dry_run: bool = False @property def gpu(self) -> int: """The primary device. Everything that addresses ONE card reads this: the bench service's `device_index`, the profiler, and the headroom check. Averaging four cards into one reading would make each of those meaningless, so a multi-device run still has a primary and the rest are held, checked and granted to the container alongside it. """ return self.gpus[0] @property def device_spec(self) -> str: """What docker's `--gpus device=` wants: every card, comma separated.""" return ",".join(str(g) for g in self.gpus) def __post_init__(self) -> None: if not self.gpus: raise ValueError("a run holds at least one device") if self.dry_run and self.namespace == ids.PRODUCTION: # A dry run never consumes a production identity. self.namespace = ids.SCRATCH # NO SKILLS OR AGENT DEFINITIONS ARE DEFAULTED BY THE DATA CLASS. # # What the agent is told is ONE document per task, # `serving_tasks//instruction.md`, and `instructions.py` only # loads it. The parser later calls `_guidance_defaults`: standard and # instructed task IDs select different, explicit skill roots. No # standalone knowledge tree exists in the plan or materializer; the # instructed model/framework notes live inside its skill. Skill fields # remain explicitly overridable for an audited experiment. # # What this lineage already learned, when there is a lineage to learn # from. A fresh one has none, and that is not a failure. if self.memory_dir is None and self.lineage_id: found = Path(self.lineage_root) / self.lineage_id / "MEMORY" self.memory_dir = found if found.is_dir() else None class Lifecycle: def __init__(self, plan: RunPlan) -> None: self.plan = plan self.identity: ids.RunIdentity | None = None self.journal: Journal | None = None self.own: Ownership | None = None self.store: CaptureStore | None = None self.publisher: Publisher | None = None self.registry: VersionRegistry | None = None self.built: ws_mod.BuiltWorkspace | None = None self.container = "" self._claimed_device_uuids: dict[int, str] | None = None self._round = 0 self._head_at_start = "" self._last_tree = "" self._last_commit = "" self._seen: set[str] = set() self._automatic_checkpoints: set[str] = set() #: A concrete version explicitly selected by the agent is a rollback #: override. The ordinary selection rule is the final working tree, #: captured automatically at run end; ``promote latest`` is therefore #: only a request to catch up, not an instruction to pin an older tree. self._explicit_head_override = "" #: Candidates whose verdict this harness could not reach. Recorded #: once each, and asked again at run end. self._withheld: set[str] = set() self._threads: list[threading.Thread] = [] #: The final validation's child process, once started. Held so a #: second caller waits for it instead of launching a rival. self._final_validation_child: subprocess.Popen | None = None #: Set once the run is wrapping up so no new capture begins after the #: remaining publication work has been joined. self._ending = threading.Event() # ------------------------------------------------------------------ create def create(self) -> ids.RunIdentity: plan = self.plan epoch = control_epoch(plan.task_dir) deadline = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=plan.hours) ).strftime("%Y-%m-%dT%H:%M:%SZ") # TIMESTAMP FIRST, so `ls` is chronological order. # # `server_output/` accumulates one directory per lineage and is # read by eye more often than by anything else, so the newest lineage # sorts last. `lineage_of` derives a lineage from the PATH a seed sits # inside, by splitting against `lineage_root`; nothing parses this # string for its parts. lineage_id = plan.lineage_id or ( f"{datetime.datetime.now(datetime.timezone.utc):%Y%m%dT%H%M%SZ}-" f"{Path(plan.task_dir).name.lower()}-" f"{agent_slug(plan.agent_model, plan.agent_effort)}") self.identity = ids.allocate( repo=plan.repo, task_id=Path(plan.task_dir).name, lineage_id=lineage_id, control_epoch=epoch, deadline_utc=deadline, namespace=plan.namespace) self.journal = Journal(self.identity.root) self.own = Ownership(self.journal) if plan.resume_run is not None: source_root = Path(plan.resume_run).resolve() source_identity = ids.load(source_root) if source_identity.lineage_id != lineage_id: raise ValueError( f"resume run {source_identity.run_id} belongs to lineage " f"{source_identity.lineage_id!r}, not {lineage_id!r}") source_journal = Journal(source_root) provider = plan.agent_cli session_id = persisted_agent_session(source_journal, provider) if not session_id: raise ValueError( f"resume run {source_identity.run_id} records no " f"{provider} session") if provider == "codex": # Codex keeps the rollout in a local session store, so both # the id and its JSONL must cross the run boundary. Claude's # OAuth-backed CLI resolves its recorded session remotely and # needs only the id; agent.launch already issues --resume. source_store = source_root / "codex-sessions" if not source_store.is_dir() or not any( session_id in path.name for path in source_store.rglob("*.jsonl")): raise ValueError( f"resume run {source_identity.run_id} does not retain " f"the rollout for Codex thread {session_id}") destination = self.identity.root / "codex-sessions" shutil.copytree(source_store, destination) plan.resume_session_id = session_id # Claude Code keeps the conversation in ~/.claude/projects INSIDE # the container, which a make-up run's new container does not have: # `--resume ` would find nothing and the run would cold-restart. # An operator who saved the old container's projects dir to # `/claude-projects` gets it copied in before the first # launch. seed = source_root / "claude-projects" self._claude_session_seed = ( seed if provider == "claude" and (seed / "-workspace" / f"{session_id}.jsonl").is_file() else None) self.journal.append( "agent_session", provider=provider, session_id=session_id, imported=True, source_run=source_identity.run_id) import_event = ("agent_session_store_imported" if provider == "codex" else "agent_session_imported") fields = { "provider": provider, "session_id": session_id, "source_run": source_identity.run_id, "source_root": str(source_root), "why": ("a separately identified make-up run continues the " "persisted provider conversation while rebuilding its " "workspace from the lineage's published head"), } if provider == "codex": fields["destination"] = str(destination) self.journal.append(import_event, **fields) lineage_dir = Path(plan.lineage_root) / lineage_id # Outside the published tree: server_output// is a claim # that a lineage exists, and this is an object database. It is keyed by # LINEAGE and not by run, because a lineage is the chain of trees -- # which is also what lets a later run carry its head forward and prove # descent from it. self.store = CaptureStore( Path(plan.lineage_root) / ".objects" / lineage_id) self.publisher = Publisher(lineage_dir) self.registry = VersionRegistry( lineage_dir, task_dir=plan.task_dir, agent={"model": plan.agent_model, "effort": plan.agent_effort}) self._head_at_start = self.publisher.head_version() # What this run was ASKED FOR, beside what it was given. # # `stop` is half the acceptance rule -- `beat-all-baselines` and # `timebox` are different experiments with the same task, card and seed # -- so it is journalled here rather than left recoverable only from # the rendered brief. # `agent_model` and `agent_effort` are here for the same reason: a # lineage is one (task, model, agent) triple, so the triple has to be # readable off the evidence rather than off whoever launched it. self.journal.append("run_created", **self.identity.to_dict(), hours=plan.hours, gpu=plan.gpu, gpus=list(plan.gpus), dry_run=plan.dry_run, stop=plan.stop, agent_cli=plan.agent_cli, agent_model=plan.agent_model, agent_effort=plan.agent_effort, resume_run=str(plan.resume_run or ""), resume_workspace=plan.resume_workspace, accepted_head_at_start=self._head_at_start) return self.identity def resume(self, *, run_id: str, task_id: str, lineage_id: str, workspace_id: str, control_epoch_id: str) -> ids.RunIdentity: """Reopen exactly one run. Every field must match; nothing is guessed.""" self.identity = ids.open_exact( repo=self.plan.repo, run_id=run_id, task_id=task_id, lineage_id=lineage_id, workspace_id=workspace_id, control_epoch=control_epoch_id, namespace=self.plan.namespace) self.journal = Journal(self.identity.root) self.own = Ownership(self.journal) lineage_dir = Path(self.plan.lineage_root) / lineage_id self.store = CaptureStore( Path(self.plan.lineage_root) / ".objects" / lineage_id) self.publisher = Publisher(lineage_dir) self.registry = VersionRegistry( lineage_dir, task_dir=self.plan.task_dir, agent={"model": self.plan.agent_model, "effort": self.plan.agent_effort}) self._head_at_start = self.publisher.head_version() self.journal.append("run_resumed", run_id=run_id, accepted_head_at_start=self._head_at_start) return self.identity # -------------------------------------------------------------- workspace def build_workspace(self) -> ws_mod.BuiltWorkspace: plan = self.plan carried: list[tuple[str, bytes]] = [] if plan.resume_workspace: source_workspace = Path(plan.resume_run) / "workspace" files = ws_mod.candidate_manifest(source_workspace) carried = [ (relative, (source_workspace / relative).read_bytes()) for relative in files ] self.journal.append( "interrupted_workspace_imported", source_run=ids.load(Path(plan.resume_run)).run_id, source_workspace=str(source_workspace), files=len(files), why=("explicit recovery retained unpublished candidate files " "after the prior run was externally interrupted; fresh " "harness controls are delivered by this run"), ) else: head_tree = self.publisher.head_tree() if head_tree and self.store.exists(head_tree): staged = self.identity.root / ".head-source" self.store.extract(head_tree, staged) for path in sorted(staged.rglob("*")): if path.is_file(): carried.append((str(path.relative_to(staged)), path.read_bytes())) # A RUN NEEDS AN OPERATING POINT BEFORE IT NEEDS A BRIEF. # # `_agentx_mvp.invocation` refuses a task whose offered concurrency is # undecided, and the verdict therefore cannot run -- but that refusal # fires at MEASUREMENT time, hours in, after the agent has optimised # for a load nobody would score it at. # # Refuse here instead: before a card is held, before an image is built, # while the fix is one calibration run. _require_an_operating_point(plan.task_dir) def text(workspace): return plan.brief or instructions_mod.build_brief( task_dir=plan.task_dir, model_id=plan.model_id, hours=plan.hours, carried=bool(carried), stop=plan.stop, workspace=workspace) self.built = ws_mod.build( identity=self.identity, task_dir=plan.task_dir, repo=plan.repo, journal=self.journal, seed=plan.seed, from_tree=carried, agents_dir=plan.agents_dir, skill_dir=plan.skill_dir, claude_project_dir=plan.claude_project_dir, ncu_skill_dir=plan.ncu_skill_dir, kernel_skill_dir=plan.kernel_skill_dir, kernelwiki_skill_dir=plan.kernelwiki_skill_dir, kernelwiki_overlay_dir=plan.kernelwiki_overlay_dir, brief=text) plan.brief = (self.built.path / "AGENT_TASK.md").read_text() self.recall_memory() move_run(self.journal, self.identity.run_id, run_state(self.journal), RunState.WORKSPACE_READY, files=self.built.delivered_files, seconds=self.built.build_seconds) return self.built def recall_memory(self) -> None: """Record that no lineage memory is delivered into the workspace. The agent sees its task, its own workspace and the evolve loop, and no previous evolve. The lineage's `memory.md` is collected at session end for the operator; it is never handed to a later agent. Journalled either way, so a run that delivered nothing and a driver that never called this differ. """ lineage_id = (self.identity.lineage_id if self.identity is not None else self.plan.lineage_id) self.journal.append( "memory_not_delivered", why=("no previous evolve is visible to the agent. The lineage " "memory is kept for the operator and is not delivered into a " "workspace"), lineage_id=lineage_id, had_memory_to_deliver=bool(self.plan.memory_dir), looked_in=(str(Path(self.plan.lineage_root) / lineage_id) if lineage_id else "")) return def persist_memory(self) -> None: """Take back whatever the agent wrote, atomically, at session end. The agent owns the content; the harness owns only that it survives and that the previous version is kept beside it. No summarisation, no ranking, no rewriting of what the agent said. """ if self.plan.memory_dir is None or self.built is None: return lineage_dir = Path(self.plan.memory_dir).parent try: from . import basic_memory report = basic_memory.collect(self.built.path, lineage_dir) except Exception as exc: # noqa: BLE001 self.journal.append("memory_persist_failed", why=repr(exc)[:400], classification=FailureClass.HARNESS.value) return self.journal.append("memory_persisted", lineage=str(lineage_dir), **report) # -------------------------------------------------------------- container @property def on_modal(self) -> bool: """Is this run's environment a Modal sandbox rather than a container on this host? Asked in one place so the guards cannot drift apart.""" return os.environ.get("SERVERBENCH_BACKEND", "").lower() == "modal" def claim_devices(self) -> dict[int, str]: """Atomically claim every clean card before any expensive setup. Model identity verification hashes the staged weights and can take tens of seconds. Leaving the fleet claim until container setup would let a card selected by the launcher be taken while that check was running. This method is idempotent so ``main`` can reserve before verification and ``start_container`` can still enforce the invariant for direct callers. """ if self.on_modal: # THE CARDS ARE NOT ON THIS MACHINE. There is nothing here to # claim, nothing to contend with, and nothing a holder process # could make visible to a neighbour -- the sandbox's GPUs are # Modal's to allocate and are exclusive by construction. Claiming # local indices would reserve someone else's cards on this host for # a run that never touches them. self.journal.append( "devices_not_local", backend="modal", why=("this run's GPUs are allocated by Modal inside its " "sandbox; the local device claim, the foreign-process " "check and the holder processes all describe this host " "and do not apply")) self._claimed_device_uuids = {} return {} if self._claimed_device_uuids is not None: return self._claimed_device_uuids if self.own is None: raise ws_mod.HarnessSetupFailed( "the run must be created before its devices can be claimed") plan = self.plan # EVERY card, not just the primary. A four-device run that checked # only its first card could start with somebody else's engine on the # other three, and a contended device is not a measurement. uuids = {index: device_uuid(index) for index in plan.gpus} for index in plan.gpus: reserved = ws_mod.container_reservations(index, uuids[index]) if reserved: raise ws_mod.HarnessSetupFailed( f"GPU {index} ({uuids[index]}) is reserved by running " f"container(s): {reserved[:3]}. An idle container may " "start GPU work again; nothing here will share its card") foreign = foreign_processes(index) if foreign: raise ws_mod.HarnessSetupFailed( f"GPU {index} ({uuids[index]}) already carries " f"{len(foreign)} compute process(es) that are not this " f"run's. A contended device is not a measurement; nothing " f"here will signal them") for index in plan.gpus: if not self.own.took_device(index, uuids[index]): raise ws_mod.HarnessSetupFailed( f"GPU {index} ({uuids[index]}) could not be claimed for " f"this run; another live owner may have selected it") self._claimed_device_uuids = uuids return uuids def start_container(self) -> str: """Compose and start the run's environment. Harbor, and only Harbor. There is no fallback to a raw `docker run`: a run that took one would be indistinguishable in the journal from one that did not, with a different set of mounts and a different uid owning the workspace. So a host that cannot drive Harbor REFUSES. `harbor/.venv` is gitignored and a fresh clone has none, so the refusal names the command that makes one. """ uuids = self.claim_devices() self.container = self._start_through_harbor(uuids) return self.container def _start_on_modal(self, python: Path) -> str: """Stand the environment up as a Modal GPU sandbox. The same contract as the Harbor path: a subprocess under an interpreter that has the SDK, a `harbor_environment.json` naming the container, and a name the rest of the lifecycle can act on. """ plan = self.plan script = plan.repo / "serverbench/scripts/agent/modal_env.py" if not script.is_file(): raise ws_mod.HarnessSetupFailed(f"{script} is missing") hours = max( 0.1, agent_mod.remaining_seconds( self.identity, paused_s=billing_mod.paused_seconds(self.journal)) / 3600.0) got = subprocess.run( [str(python), str(script), "--workspace", str(self.built.path), "--task", str(plan.task_dir), "--out", str(self.identity.root), "--hours", f"{hours:.3f}", "--agent-cli", plan.agent_cli, "--agent-model", plan.agent_model or "", "--agent-effort", plan.agent_effort or "", *(["--agent-env", str(plan.agent_env)] if plan.agent_env else []), *(["--credentials", str(plan.credentials)] if getattr(plan, "credentials", None) else [])], cwd=str(plan.repo), capture_output=True, text=True, timeout=3600, check=False) if got.stdout: print(got.stdout[-3000:]) if got.returncode != 0: raise ws_mod.HarnessSetupFailed( f"modal_env.py exited {got.returncode}: " f"{(got.stdout + got.stderr)[-800:]}") record = self.identity.root / "harbor_environment.json" try: container = json.loads(record.read_text("utf-8"))["container"] except Exception as exc: # noqa: BLE001 raise ws_mod.HarnessSetupFailed( f"the Modal environment named no container: {exc!r}") from None self.journal.append("container_backend", **backend.describe(container)) self.own.took_container(container) return container def _start_through_harbor(self, uuids: dict) -> str: """Stand the environment up with Harbor, or raise. Returns its name. Harbor runs as a SUBPROCESS under `harbor/.venv/bin/python`, never imported here: it declares 22 runtime dependencies -- pydantic, litellm, datasets, fastapi, supabase, and `requests`/`httpx`, which are already forbidden in the core -- and needs Python 3.12 where the harness promises a 3.10 floor. The core has to import inside a bare image to verify a candidate there, so `harbor` is on `tests/repo/test_the_core_imports_in_a_bare_image.py`'s FORBIDDEN list; that check cannot see transitive imports, and without the name an `import harbor` would drag two forbidden packages in with the checker green. Every failure below raises `HarnessSetupFailed`, which `main()` turns into a typed `HARNESS_SETUP_FAILED` exit. None of them falls back. """ plan = self.plan python = plan.repo / "harbor" / ".venv" / "bin" / "python" # ONE BACKEND PER ENVIRONMENT KIND, chosen here and nowhere else. # `modal_env.py` writes the same `harbor_environment.json` in the same # shape and names its container `modal:`, so everything below # this line is identical for both. if self.on_modal: return self._start_on_modal(python) script = plan.repo / "serverbench/scripts/agent/harbor_env.py" if not python.is_file(): raise ws_mod.HarnessSetupFailed( f"no Harbor interpreter at {python}. Harbor composes and " f"starts every run's environment and there is no second way " f"to do it; `harbor/.venv` is gitignored, so run " f"`cd harbor && uv sync` on this host") if not script.is_file(): raise ws_mod.HarnessSetupFailed(f"{script} is missing") got = subprocess.run( [str(python), str(script), "--workspace", str(self.built.path), "--model", str(plan.model_dir), "--env-input", str(plan.env_input), "--image", plan.image, "--gpu", plan.device_spec, "--out", str(self.identity.root), "--task", str(plan.task_dir), "--agent-cli", plan.agent_cli, "--agent-model", plan.agent_model, "--agent-effort", plan.agent_effort, *(["--agent-env", str(self.plan.agent_env)] if self.plan.agent_env else []), *(["--credentials", str(plan.credentials)] if plan.credentials else []), "--session-id", f"sb{self.identity.run_id.replace('-', '')[-20:]}"], cwd=str(plan.repo), capture_output=True, text=True, timeout=1800, check=False) if got.stdout: print(got.stdout[-2000:]) if got.returncode != 0: raise ws_mod.HarnessSetupFailed( f"harbor_env.py exited {got.returncode}: " f"{(got.stdout + got.stderr)[-600:]}") record = self.identity.root / "harbor_environment.json" try: composed = json.loads(record.read_text("utf-8")) container = composed["container"] except Exception as exc: # noqa: BLE001 raise ws_mod.HarnessSetupFailed( f"harbor reported success and named no container: {exc!r}" ) from None # CLAIM THE BROKER. It is started detached so it survives the setup # subprocess and keeps relaying while the agent works, so it must be # claimed or `release_all` cannot stop it and brokers accumulate, one # port each on the docker bridge. `Ownership` is the right place rather # than a kill in teardown, because a run may only signal what its # journal says it took. broker_pid = ((composed.get("egress_broker") or {}).get("pid") or 0) if broker_pid: self.own.took_process( int(broker_pid), what="egress broker (the container's only route out)", argv=[str((composed.get("egress_broker") or {}).get("endpoint") or "")]) if not container_is_up(container): raise ws_mod.HarnessSetupFailed( f"harbor's container {container} is not running") self.journal.append("container_backend", backend="harbor", container=container, image=plan.image, **_image_identity(plan.image), driver="harbor.environments.docker.DockerEnvironment") self.own.took_container(container) self._hold_cards() return container #: How much each holder reserves, over and above its CUDA context. Small on #: purpose: it is memory the agent cannot use, and on a task whose whole #: point is KV capacity that cost has to be tiny and stated. HOLD_MIB = 256 def _hold_cards(self) -> None: """Make this run's cards LOOK held to everyone, not just to us. Skipped entirely when the cards are not on this machine: see `claim_devices`. A card this run owns reads 0 MiB for most of the run -- the agent spends its first hour writing code, and the card is idle again between every teardown and restart of its server. Three mechanisms say it is taken and none is visible to another user: docker's `device_ids` governs only what our container sees, and the claims and lock directories are read by ServerBench tooling alone. On a shared box a neighbour can pick a card in the window where ours looks empty. A small allocation puts the reservation in the one place everybody actually reads, `nvidia-smi`. Recorded with `took_process`, so the run releases it the same way it releases everything else it started -- and released BEFORE the final verdict, because the verdict needs the card clear. `--parent-pid` is a second, independent guarantee: a holder whose driver is SIGKILLed exits on its own rather than becoming stale GPU memory. Never fatal. A host with no torch-capable interpreter simply does not get the reservation, and the run says so. """ if self.on_modal: return interpreter = ws_mod.holder_interpreter() script = plan_repo = self.plan.repo / "serverbench/scripts/gpu_holder.py" if not interpreter or not script.is_file(): self.journal.append( "cards_not_held", why=("no interpreter on this host can import torch and see a " "GPU, so the cards are claimed but will not look busy to " "a neighbour reading nvidia-smi" if not interpreter else f"{plan_repo} is missing")) return for index in self.plan.gpus: try: argv = [interpreter, str(script), "--device", str(index), "--mib", str(self.HOLD_MIB), "--parent-pid", str(os.getpid())] # ITS OUTPUT IS EVIDENCE, NOT NOISE. # # The holder prints every take and every release; that is the # record needed to say whether a measurement was taken on a # clean card. log = self.identity.root / f"gpu-holder.{int(index)}.log" handle = log.open("a", encoding="utf-8") proc = subprocess.Popen( # noqa: S603 argv, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True) self.own.took_process(proc.pid, "gpu-holder", argv) self.journal.append( "card_held", index=int(index), pid=proc.pid, mib=self.HOLD_MIB, log=str(log), note=("holds only until the card is first genuinely busy, " "then releases and exits -- a holder resident on a " "card the agent is benchmarking on changes what the " "benchmark measures")) except (OSError, subprocess.SubprocessError) as exc: self.journal.append("cards_not_held", index=int(index), why=f"{type(exc).__name__}: {exc}"[:200]) def _record_in_the_ledger(self) -> None: """Append this run's verdict to its task's CSV under evaluation_progress. ONE TASK, ONE FILE, so two tasks never share a column and a number is never compared across hardware by accident. Rows are ordered by `dollars_per_dollar`, which is the objective the suite exists to move. Reads only what the verdict already wrote -- FINAL.json and `scenario-candidate.json` -- so this cannot disagree with the verdict. A repeat of the same configuration is a NEW row, keyed by run id, not a replacement: the second measurement of a setting is the thing you compare the first against. Never fatal, and last. Every piece of evidence is already on disk by the time this runs; losing a spreadsheet row because a path moved must not cost the record of what happened. """ if self.identity is None or self.plan.dry_run: return recorder = (self.plan.repo / "evaluation_progress" / "record_run.py") if not recorder.is_file(): return try: got = subprocess.run( [sys.executable, str(recorder), "--run", str(self.identity.root)], cwd=str(self.plan.repo), capture_output=True, text=True, timeout=300, check=False) except (OSError, subprocess.SubprocessError) as exc: self.journal.append("ledger_not_updated", why=describe(exc)) return line = (got.stdout or got.stderr or "").strip()[:300] self.journal.append( "ledger_updated" if got.returncode == 0 else "ledger_not_updated", detail=line) def _release_holders(self, why: str) -> None: """Let go of the cards, by exact recorded pid. Called BEFORE the final verdict: the verdict refuses a card carrying a compute process it does not own, and a holder is exactly that from its point of view. Called again on teardown, which is harmless -- a pid already gone is recorded as such. """ for event in self.own.processes: if event.get("what") != "gpu-holder": continue pid = int(event.get("pid") or 0) if not pid: continue try: os.kill(pid, signal.SIGTERM) how = "signal 15" except ProcessLookupError: how = "already gone" except OSError as exc: how = f"{type(exc).__name__}: {exc}"[:80] self.journal.append("card_released", pid=pid, how=how, why=why) def boot(self) -> ws_mod.BootReport: report = ws_mod.boot_contract( workspace=self.built.path, container=self.container, python="python3", journal=self.journal, withheld=("oracle", "references-qualification.json", "references-final.json", "serving_evolve")) if not report.ok: self.journal.append("harness_setup_failed", why=report.failure()) return report # ------------------------------------------------------------- development #: How often to re-check the cards for somebody else's work, in seconds. #: Cheap -- one `nvidia-smi` query -- and the window it closes is hours #: wide, so it does not need to be frequent. CONTENTION_EVERY_S = 300.0 #: How often the run re-copies the operator's credentials into its own. #: #: Short, because the window this closes is measured in seconds: a host #: token refresh revokes the token the container holds, and the agent's #: next request fails. CREDENTIALS_EVERY_S = 60.0 def _refresh_credentials(self, elapsed: float) -> None: """Keep the container's token the same token the host is using. The credentials reach the container as a bind of ONE FILE, and a file bind follows the inode it resolved at container start. Claude Code refreshes its OAuth token by atomic replace -- new inode -- and the refresh REVOKES the previous token. So without this, any refresh by anything else logged in as this operator kills the run with a 401. `harbor_env.stage_credentials` gives the run its own copy at an inode it owns, and this rewrites that copy IN PLACE, so the container -- which is holding exactly that inode -- sees the new bytes without a remount. Nothing about the VALUE is journalled, only that a refresh happened. """ if self.plan.agent_cli != "claude": return import time as _time now = _time.monotonic() last = getattr(self, "_credentials_checked", 0.0) if now - last < self.CREDENTIALS_EVERY_S: return self._credentials_checked = now from lifecycle import credentials as creds_mod source = creds_mod.source_path() staged = creds_mod.staged_path(self.identity.root) if not staged.is_file(): return try: changed = creds_mod.refresh(staged, source) except Exception as exc: # noqa: BLE001 self.journal.append("credentials_refresh_failed", why=repr(exc)[:200], elapsed_seconds=round(elapsed, 1)) return if not changed: return # A SANDBOX IS NOT HOLDING THIS INODE. # # The in-place rewrite above is the whole fix on docker, where the file # is a bind of exactly that inode. On Modal there are no bind mounts: # `modal_env.py` copied the token in once at setup, so rewriting the # host file reaches nothing and the sandbox keeps a token the refresh # has just revoked. # # Pushed through `backend.put_file`, which sends the bytes on stdin -- # never on a command line -- and chowns the file to the agent at 0600, # exactly as the docker path leaves it. if self.container and backend.is_modal(self.container): try: backend.put_file(self.container, staged.read_text("utf-8"), creds_mod.IN_CONTAINER) except Exception as exc: # noqa: BLE001 self.journal.append( "credentials_not_pushed_to_sandbox", elapsed_seconds=round(elapsed, 1), why=repr(exc)[:200], because=("the sandbox still holds the token the host " "refresh revoked, so the agent's next request " "will fail with 401 OAuth access token has been " "revoked")) return self.journal.append( "credentials_refreshed", elapsed_seconds=round(elapsed, 1), backend="modal", why=("the operator's token changed on the host and the new " "bytes were written into the sandbox, which holds a copy " "rather than a bind and would otherwise keep using a " "token the refresh had revoked")) return self.journal.append( "credentials_refreshed", elapsed_seconds=round(elapsed, 1), why=("the operator's token changed on the host and was copied " "into this run's own file in place. The container holds " "that inode, so its next request uses the new token " "instead of one the refresh had revoked")) def _watch_for_contention(self, elapsed: float) -> None: """Record a foreign process that arrives on our card MID-RUN. `start_container` refuses a card that already carries somebody else's compute, but a card that is clean when a run starts can be shared an hour later. OWNERSHIP IS DECIDED BY THE CONTAINER, not by `foreign_processes`, which is a run-START question ("nothing of ours is on the card yet"). Mid-run it would record the AGENT'S OWN SERVER, which restarts with a new pid every time the agent iterates, and a check that fires constantly is a check nobody reads. This does NOT stop the run, and deliberately so: the process is not ours to signal, the agent may be mid-benchmark, and killing a run over a neighbour's arrival throws away hours of good work for a fact that can simply be recorded. What it does is make the window a FACT about the run -- so a number measured inside it is known to have been, rather than quietly averaged in with the rest. """ if self.on_modal: # THE CARDS ARE NOT ON THIS MACHINE. Watching local indices here # would report a neighbour's process on this host as contention on # a run whose GPUs are Modal's. return import time as _time now = _time.monotonic() last = getattr(self, "_contention_checked", 0.0) if now - last < self.CONTENTION_EVERY_S: return self._contention_checked = now for index in self.plan.gpus: try: foreign = ws_mod.processes_not_in( int(index), self.container or "", # PROCESSES THIS RUN STARTED ARE OURS, container or not. # # Ownership by container alone is not enough: the GPU # holders run on the HOST by necessity, because the harness # interpreter carries no torch, and would otherwise be # reported as neighbours. The ownership register the # driver keeps answers exactly this question. mine=tuple(str(e.get("pid")) for e in self.own.processes)) except Exception: # noqa: BLE001 continue # never fatal if not foreign: continue seen = getattr(self, "_contention_seen", set()) key = (int(index), tuple(sorted(foreign))) if key in seen: continue seen.add(key) self._contention_seen = seen full_test_thread = getattr(self, '_development_full_test_thread', None) if full_test_thread is not None and full_test_thread.is_alive(): self.journal.append( 'device_activity_during_development_full_test', index=int(index), elapsed_seconds=round(elapsed, 1), outside_agent_container=foreign[:8], why=('the full verifier runs sibling containers on this allocation; ' 'being outside the agent container alone does not establish foreign contention')) continue self.journal.append( "device_contended", index=int(index), elapsed_seconds=round(elapsed, 1), foreign=foreign[:8], why=("a process that is not this run's appeared on a card this " "run holds, after the start-up check had passed. The run " "continues -- the process is not ours to signal and the " "agent may be mid-benchmark -- but every measurement from " "here shares the card, so this window is recorded rather " "than averaged in")) def develop(self) -> agent_mod.AgentRun: move_run(self.journal, self.identity.run_id, run_state(self.journal), RunState.AGENT_RUNNING) def tick(elapsed: float) -> None: # None of these may take the agent down with them, and none may # fail silently. A swallowed boundary is invisible: the run looks # healthy and simply never produces a version. self.take_requested_boundaries(elapsed) self.take_requested_promotions(elapsed) self.take_evidence_boundaries(elapsed) self._watch_for_contention(elapsed) self._refresh_credentials(elapsed) # THE CONTINUATION LOOP. # # An agent that stops with budget left is resumed into the SAME # session, the same workspace and the same original deadline, so an # agent that exits early does not end the run as COMPLETED with its # budget unspent. # # What this is NOT: a supervisor. There is no new run, no new lineage, # no new budget and no new agent. `remaining_seconds` is computed from # the deadline set at creation, so continuing can never buy time. run: agent_mod.AgentRun | None = None cold_next = False idle_next = False idle_streak = 0 self._seed_claude_session() while True: run = agent_mod.launch( identity=self.identity, journal=self.journal, container=self.container, brief=self.plan.brief, # The task owns every word an agent reads, including the # continuation and restart prompts. See # `lifecycle/instructions.py`. task_dir=self.plan.task_dir, model_flag=f"--model {self.plan.agent_model}", effort_flag=(f"--effort {self.plan.agent_effort}" if self.plan.agent_effort else ""), agent_cli=self.plan.agent_cli, agent_model=self.plan.agent_model, agent_effort=self.plan.agent_effort, on_tick=tick, cold_start=cold_next, idle_restart=idle_next, resume_session_id=( self.plan.resume_session_id if not self.journal.last("agent_launched") else "")) cold_next = False idle_next = False if not run.stopped_early: return run # A CONFIGURATION FAILURE IS NOT AN AGENT THAT STOPPED. # # The loop below exists for an agent with budget left -- a crash, a # context exhaustion, a dropped stream -- and asking again is right # for all of those. It is wrong when the agent never reached its # model: every attempt fails exactly like the first, and the # run ends up reporting something about the AGENT for a defect in # its own command line. See `agent.terminal_failure`. if run.terminal_because: # RUNNING OUT OF MONEY IS A PAUSE, NOT A VERDICT. # # Every other terminal cause here is a configuration error that # no one is coming to fix mid-run: the model does not exist, no # credential reached the container. A spend limit is different # in exactly one way that matters -- an operator CAN fix it, # from a dashboard, in under a minute. Ending the run instead # discards the workspace, the lineage position and every # remaining GPU-hour on cards already paid for. # # So the run parks. See `lifecycle/billing.py` for what ends # the wait (the balance recovering, or the credentials file # changing -- a REPLACED key is a different token whose old # balance never recovers, so the file has to be watched too). # The pause is added back to the deadline, because `--hours` # bounds the AGENT's work and a parked run has no agent. if billing_mod.is_billing(run.terminal_because): parked = billing_mod.wait_for_credit( secrets_dir=self.identity.root / "secrets", journal=self.journal) if parked.get("resumed"): continue self.journal.append( "agent_cannot_start", why=run.terminal_because, session_id=run.session_id, launches=len(self.journal.events("agent_launched")), remaining_seconds=round( agent_mod.remaining_seconds( self.identity, paused_s=billing_mod.paused_seconds(self.journal)), 1)) return run left = agent_mod.remaining_seconds( self.identity, paused_s=billing_mod.paused_seconds(self.journal)) launches = len(self.journal.events("agent_launched")) if launches >= MAX_LAUNCHES: self.journal.append( "continuation_declined", why=( f"{launches} launches in this run, which is the " f"runaway ceiling of {MAX_LAUNCHES}"), launches=launches, remaining_seconds=round(left, 1)) return run # A SESSION LIMIT IS A CLOCK, NOT A CRASH. Park, do not relaunch. # # An agent that exits with "You've hit your session limit - resets #