"""Own one hardened agent-server container per conversation.""" from __future__ import annotations import asyncio import hashlib import os import subprocess import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager, suppress from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Final from urllib.error import URLError from urllib.request import urlopen from uuid import UUID, uuid4 from openhands.agent_server.config import V1_SESSION_API_KEY_ENV, Config from openhands.agent_server.conversation_registry import ConversationRegistry from openhands.agent_server.docker_runtime.provisioning import RuntimeProvisioningStore from openhands.agent_server.docker_runtime.storage import DockerRuntimeStorage from openhands.agent_server.launch import container_browser_enabled from openhands.agent_server.models import ( ConversationRuntimeInfo, ConversationRuntimeStatus, ) from openhands.agent_server.persistence.store import _get_persistence_dir from openhands.agent_server.storage import Reclaimer from openhands.sdk.conversation.request import ConversationContainerOptions from openhands.sdk.conversation.state import ConversationExecutionStatus from openhands.sdk.logger import get_logger from openhands.sdk.utils.cipher import Cipher from openhands.sdk.utils.command import execute_command, sanitized_env if TYPE_CHECKING: from fastapi import APIRouter from openhands.agent_server.conversation_service import ConversationService logger = get_logger(__name__) _CONVERSATIONS_DIR: Final[str] = "/var/openhands/conversations" _PERSISTENCE_DIR: Final[str] = "/var/openhands/.openhands" _WORKSPACE_DIR: Final[str] = "/workspace" _OWNER_LABEL: Final[str] = "ai.openhands.runtime-owner" # Environment variables read by openhands.sdk.observability (Laminar / OTEL # tracing). Only those actually set on the host are forwarded into the # conversation container, so an unconfigured host injects nothing. _OBSERVABILITY_ENV_VARS: Final[tuple[str, ...]] = ( "LMNR_PROJECT_API_KEY", "LMNR_BASE_URL", "LMNR_HTTP_PORT", "LMNR_GRPC_PORT", "LMNR_FORCE_HTTP", "LMNR_INSTRUMENTS", "LMNR_SPAN_CONTEXT", "OTEL_ENDPOINT", "OTEL_EXPORTER", "OTEL_HEADERS", "OTEL_PROTOCOL", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OPENHANDS_OBSERVABILITY_METADATA", "OPENHANDS_OBSERVABILITY_TAGS", "OPENHANDS_OBSERVABILITY_SPAN_NAME", "OPENHANDS_OBSERVABILITY_PARENT_SPAN_CONTEXT", ) class RuntimeArchivedError(RuntimeError): """Retention deleted the runtime; the conversation is read-only.""" @dataclass(slots=True) class ConversationContainer: host: str api_key: str container_id: str def stop(self) -> None: result = execute_command(["docker", "stop", self.container_id]) if result.returncode != 0 and "No such container" not in result.stderr: raise RuntimeError( f"Failed to stop conversation container: {result.stderr}" ) def is_running(self) -> bool: result = execute_command( ["docker", "inspect", "-f", "{{.State.Running}}", self.container_id] ) return result.returncode == 0 and result.stdout.strip() == "true" class DockerConversationRegistry(ConversationRegistry): def __init__(self, config: Config) -> None: super().__init__(config) paths = ( f"{config.conversations_path.resolve()}\0" f"{_get_persistence_dir(config).resolve()}" ) self.owner = hashlib.sha256(paths.encode()).hexdigest()[:24] self.provisioning = RuntimeProvisioningStore(config) self._containers: dict[UUID, ConversationContainer] = {} self._starts: dict[UUID, asyncio.Task[ConversationContainer]] = {} self._deleting: set[UUID] = set() self._lock = asyncio.Lock() self._service: ConversationService | None = None self._last_access: dict[UUID, float] = {} self._sessions: dict[UUID, int] = {} self._eviction_task: asyncio.Task[None] | None = None self.reclaimer = Reclaimer( DockerRuntimeStorage(self), disk_budget=config.conversation_storage_disk_budget, retention_days=config.conversation_storage_retention_days, ) def configure_service(self, service: ConversationService) -> None: self._service = service service.runtime_cipher_resolver = self.resolve_persisted_cipher async def refresh_conversation(self, conversation_id: UUID) -> None: """Re-read the conversation from disk after its runtime changed.""" if self._service is not None: await self._service.refresh_persisted_conversation(conversation_id) def resolve_persisted_cipher(self, conversation_id: UUID) -> Cipher: """Resolve persisted state without weakening per-runtime isolation. Conversations created before Docker mode have no provisioning identity and were encrypted with the host key. A present but invalid identity still raises rather than falling back to that key. """ identity = self.provisioning.load_optional(conversation_id) return identity.cipher if identity is not None else self.provisioning.cipher def archived_marker(self, conversation_id: UUID) -> Path: # Beside the manifest, not in it: the manifest must stay loadable by # older builds, and it holds the key to the conversation's history. return self.provisioning.control_root / f"{conversation_id.hex}.archived.json" def is_archived(self, conversation_id: UUID) -> bool: return self.archived_marker(conversation_id).exists() def runtime_info(self, conversation_id: UUID) -> ConversationRuntimeInfo: identity = self.provisioning.load_optional(conversation_id) if identity is None or self.is_archived(conversation_id): return ConversationRuntimeInfo( runtime_status=ConversationRuntimeStatus.MISSING, can_resume=False, ) return ConversationRuntimeInfo( runtime_status=( ConversationRuntimeStatus.AVAILABLE if self.get(conversation_id) else ConversationRuntimeStatus.STARTING if self.is_starting(conversation_id) else ConversationRuntimeStatus.MISSING ), can_resume=True, ) def serves_persisted_event_reads(self, _conversation_id: UUID) -> bool: return True async def start(self) -> None: await asyncio.to_thread(self.cleanup_stale_containers) # After the cleanup: with no container left, every runtime is reclaimable. await self.reclaimer.start() if self.config.conversation_idle_ttl_seconds: self._eviction_task = asyncio.create_task(self._evict_idle_runtimes_loop()) def add_execution_routes(self, router: APIRouter) -> None: from openhands.agent_server.docker_runtime.routers import ( docker_conversation_router, ) from openhands.agent_server.event_router import event_read_router # Persisted event history is safe to read from the outer catalog for # both Docker-backed and historical host-local conversations. Writes # continue through the runtime proxy below. router.include_router(event_read_router) router.include_router(docker_conversation_router) @property def workspace_router(self) -> APIRouter: from openhands.agent_server.docker_runtime.routers import ( docker_workspace_router, ) return docker_workspace_router @property def conversation_sockets_router(self) -> APIRouter: from openhands.agent_server.docker_runtime.routers import docker_sockets_router return docker_sockets_router @property def session_sockets_router(self) -> APIRouter: from openhands.agent_server.docker_runtime.routers import ( docker_session_sockets_router, ) return docker_session_sockets_router def conversation_dir(self, conversation_id: UUID) -> Path: return self.provisioning.direct_child( self.config.conversations_path, conversation_id.hex ) def workspace_dir(self, conversation_id: UUID) -> Path: workspace = self.provisioning.load(conversation_id).workspace_path if workspace.is_symlink(): raise ValueError("Conversation workspace must not be a symlink") return workspace.resolve() def get(self, conversation_id: UUID) -> ConversationContainer | None: return self._containers.get(conversation_id) def is_starting(self, conversation_id: UUID) -> bool: return conversation_id in self._starts @asynccontextmanager async def runtime_idle(self, conversation_id: UUID) -> AsyncIterator[bool]: """Hold the lifecycle lock; True if no container runs or starts for the runtime and it is not being deleted. Nothing can start a container until the block exits, so files moved inside it are not in use. """ async with self._lock: yield not ( self.get(conversation_id) or self.is_starting(conversation_id) or conversation_id in self._deleting ) def attach_session(self, conversation_id: UUID) -> None: """Record an outer proxied session attached to this runtime. Non-zero counts suppress idle eviction, mirroring the inner ``EventService.has_external_subscribers()`` guard: a client holding a live events websocket or a long-lived proxied stream keeps the container alive even while the conversation itself looks idle. """ self._sessions[conversation_id] = self._sessions.get(conversation_id, 0) + 1 def detach_session(self, conversation_id: UUID) -> None: """Release a session recorded by :meth:`attach_session`.""" remaining = self._sessions.get(conversation_id, 0) - 1 if remaining > 0: self._sessions[conversation_id] = remaining else: self._sessions.pop(conversation_id, None) self._last_access[conversation_id] = time.monotonic() def has_attached_sessions(self, conversation_id: UUID) -> bool: """True if an outer proxied session is currently attached.""" return self._sessions.get(conversation_id, 0) > 0 def cleanup_stale_containers(self) -> None: result = execute_command( ["docker", "ps", "-aq", "--filter", f"label={_OWNER_LABEL}={self.owner}"] ) if result.returncode != 0: logger.warning("Failed to list stale conversation containers") return ids = result.stdout.split() if ids: execute_command(["docker", "rm", "-f", *ids]) async def get_or_create(self, conversation_id: UUID) -> ConversationContainer: async with self._lock: if conversation_id in self._deleting: raise RuntimeError("Conversation is being deleted") if self.is_archived(conversation_id): raise RuntimeArchivedError("Conversation runtime was archived") self._last_access[conversation_id] = time.monotonic() container = self._containers.get(conversation_id) if container is not None: if await asyncio.to_thread(container.is_running): return container async with self._lock: if self._containers.get(conversation_id) is container: self._containers.pop(conversation_id) async with self._lock: if conversation_id in self._deleting: raise RuntimeError("Conversation is being deleted") task = self._starts.get(conversation_id) if task is None: task = asyncio.create_task( asyncio.to_thread(self._build_container, conversation_id) ) self._starts[conversation_id] = task try: container = await asyncio.shield(task) except BaseException: async with self._lock: if self._starts.get(conversation_id) is task: self._starts.pop(conversation_id, None) raise async with self._lock: existing = self._containers.get(conversation_id) if existing is not None: if existing is not container: await asyncio.to_thread(container.stop) self._last_access[conversation_id] = time.monotonic() return existing if self._starts.get(conversation_id) is not task: await asyncio.to_thread(container.stop) raise RuntimeError("Conversation container start was cancelled") self._starts.pop(conversation_id, None) self._containers[conversation_id] = container self._last_access[conversation_id] = time.monotonic() return container async def begin_delete(self, conversation_id: UUID) -> bool: async with self._lock: if conversation_id in self._deleting: return False self._deleting.add(conversation_id) return True async def finish_delete(self, conversation_id: UUID) -> None: async with self._lock: self._deleting.discard(conversation_id) async def stop(self, conversation_id: UUID) -> None: async with self._lock: task = self._starts.pop(conversation_id, None) container = self._containers.pop(conversation_id, None) self._last_access.pop(conversation_id, None) self._sessions.pop(conversation_id, None) if task is not None: try: started = await task except Exception: started = None container = container or started if container is not None: await asyncio.to_thread(container.stop) await self.reclaimer.on_stop(conversation_id) async def shutdown(self) -> None: if self._eviction_task is not None: self._eviction_task.cancel() with suppress(asyncio.CancelledError): await self._eviction_task self._eviction_task = None ids = set(self._containers) | set(self._starts) await asyncio.gather(*(self.stop(cid) for cid in ids), return_exceptions=True) # Last, so it also covers the stops above; the next start sweeps them. await self.reclaimer.shutdown() async def _evict_idle_runtimes_loop(self) -> None: ttl = self.config.conversation_idle_ttl_seconds if not ttl: return interval = max(1.0, min(60.0, ttl / 2)) while True: await asyncio.sleep(interval) try: await self._evict_idle_runtimes(ttl) except Exception: logger.exception("error_evicting_idle_docker_runtimes") async def _evict_idle_runtimes(self, ttl_seconds: float) -> None: service = self._service if service is None: return cutoff = time.monotonic() - ttl_seconds async with self._lock: candidates = [ (conversation_id, container) for conversation_id, container in self._containers.items() if self._last_access.get(conversation_id, float("inf")) <= cutoff and not self.has_attached_sessions(conversation_id) ] for conversation_id, container in candidates: info = await service.get_conversation(conversation_id) if ( info is None or info.execution_status == ConversationExecutionStatus.RUNNING ): continue async with self._lock: if self._containers.get(conversation_id) is not container: continue if self._last_access.get(conversation_id, float("inf")) > cutoff: continue if self.has_attached_sessions(conversation_id): continue self._containers.pop(conversation_id) self._last_access.pop(conversation_id, None) try: await asyncio.to_thread(container.stop) except Exception: async with self._lock: if conversation_id not in self._containers: self._containers[conversation_id] = container self._last_access[conversation_id] = time.monotonic() logger.warning( "Failed to stop idle conversation runtime %s", conversation_id, exc_info=True, ) else: logger.info( "Stopped idle conversation runtime %s (idle >= %.0fs)", conversation_id, ttl_seconds, ) await service.refresh_persisted_conversation(conversation_id) await self.reclaimer.on_stop(conversation_id) def _build_container(self, conversation_id: UUID) -> ConversationContainer: identity = self.provisioning.load(conversation_id) options = identity.container or ConversationContainerOptions() runtime_dir = self.provisioning.runtime_dir(conversation_id) persistence_dir = self.provisioning.direct_child(runtime_dir, "persistence") conversation_dir = self.conversation_dir(conversation_id) workspace_dir = self.workspace_dir(conversation_id) for directory in (persistence_dir, conversation_dir, workspace_dir): directory.mkdir(parents=True, mode=0o700, exist_ok=True) env = sanitized_env() env.update( { "HOME": _PERSISTENCE_DIR, "OH_CONVERSATIONS_PATH": _CONVERSATIONS_DIR, "OH_PERSISTENCE_DIR": _PERSISTENCE_DIR, "OH_CONVERSATION_RUNTIME": "local", "OH_ENABLE_BROWSER": ( "1" if container_browser_enabled(self.config) and options.browser else "0" ), "OH_SECRET_KEY": identity.encryption_key.get_secret_value(), V1_SESSION_API_KEY_ENV: identity.api_key.get_secret_value(), "OH_RUNTIME_LAUNCHED_PROFILE": ( identity.launched_agent_profile.model_dump_json() if identity.launched_agent_profile else "" ), } ) if "DEBUG" in os.environ: env["DEBUG"] = os.environ["DEBUG"] # Forward host observability configuration so the in-container agent # server initializes tracing with the same Laminar/OTEL backend. for name in _OBSERVABILITY_ENV_VARS: value = os.environ.get(name) if value: env[name] = value else: env.pop(name, None) flags: list[str] = [] forwarded_env_names: list[str] = [ "HOME", "OH_CONVERSATIONS_PATH", "OH_PERSISTENCE_DIR", "OH_CONVERSATION_RUNTIME", "OH_ENABLE_BROWSER", "OH_SECRET_KEY", V1_SESSION_API_KEY_ENV, "OH_RUNTIME_LAUNCHED_PROFILE", "DEBUG", ] forwarded_env_names.extend( name for name in _OBSERVABILITY_ENV_VARS if name in env ) for name in forwarded_env_names: if name in env: flags.extend(("-e", name)) for host, target in ( (conversation_dir, f"{_CONVERSATIONS_DIR}/{conversation_id.hex}"), (persistence_dir, _PERSISTENCE_DIR), (workspace_dir, _WORKSPACE_DIR), ): flags.extend(("-v", f"{host}:{target}")) memory = options.memory or self.config.conversation_container_memory if memory: flags.extend(("--memory", memory)) cpus = ( options.cpus if options.cpus is not None else self.config.conversation_container_cpus ) if cpus is not None: flags.extend(("--cpus", str(cpus))) if self.config.conversation_container_pids_limit is not None: flags.extend( ("--pids-limit", str(self.config.conversation_container_pids_limit)) ) command = [ "docker", "run", "-d", "--rm", "--user", f"{os.getuid()}:{os.getgid()}", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", # Core dumps land in the bind-mounted workspace at 1-2 GB each. "--ulimit", "core=0", "--add-host", "host.docker.internal:host-gateway", "--label", f"{_OWNER_LABEL}={self.owner}", "--name", f"agent-server-conversation-{uuid4()}", "-p", "127.0.0.1::8000", *flags, self.config.conversation_image, "--host", "0.0.0.0", "--port", "8000", ] result = subprocess.run( command, env=env, capture_output=True, text=True, check=False ) if result.returncode != 0: raise RuntimeError(result.stderr.strip() or "docker run failed") container_id = result.stdout.strip() try: binding = execute_command(["docker", "port", container_id, "8000/tcp"]) address, port = binding.stdout.strip().rsplit(":", 1) if binding.returncode != 0 or address != "127.0.0.1": raise RuntimeError("Docker did not create a loopback port binding") container = ConversationContainer( host=f"http://127.0.0.1:{int(port)}", api_key=identity.api_key.get_secret_value(), container_id=container_id, ) self._wait_until_ready(container) return container except BaseException: execute_command(["docker", "stop", container_id]) raise def _wait_until_ready(self, container: ConversationContainer) -> None: deadline = time.monotonic() + self.config.conversation_container_startup_timeout while time.monotonic() < deadline: try: with urlopen(container.host + "/health", timeout=1) as response: if 200 <= response.status < 300: return except (URLError, TimeoutError, ConnectionError): pass running = execute_command( [ "docker", "inspect", "-f", "{{.State.Running}}", container.container_id, ] ) if running.stdout.strip() != "true": raise RuntimeError("Conversation container stopped during startup") time.sleep(1) raise RuntimeError("Conversation container failed to become healthy in time")