// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 //! Runtime invocation helpers. use std::collections::{BTreeMap, BTreeSet}; use std::ffi::OsString; use std::fs::File; use std::io::{BufRead, BufReader, ErrorKind, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, ChildStdin, Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; use std::sync::{Arc, LazyLock, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use crate::agent_config::validate_agent_config; use crate::agent_execution::{ AgentArtifact, AgentRunError, AgentRunRequest, AgentRunResult, AgentRunStatus, AgentUsage, }; use crate::config::{ AdapterKind, AgentConfig, CapabilityPlan, CapabilityTarget, ControlLocation, EnvironmentOwnership, RunPlan, TelemetryPlan, is_one_folder_name, validate_adapter_config_compatibility, validate_agent_config_extensions, validate_agent_run_request_extensions, validate_agent_run_result_extensions, validate_config, validate_harness_settings, validate_tool_definitions, validate_workflow, }; use crate::environment::resolve_environment_provider; use crate::error::{FabricError, Result}; use crate::resource_snapshot::{LocalHostProcess, RuntimeResourceSnapshot, snapshot_local_host}; #[cfg(test)] use crate::system_stats::ResourceObservation; use crate::system_stats::{CgroupStats, InvocationSystemStats, StatsSession, SystemReader}; mod openshell; use openshell::OpenShellRuntimeAdapter; pub(crate) fn remote_check_agent( environment: &EnvironmentHandle, runtime_id: &str, ) -> Result> { openshell::check_agent(environment, runtime_id) } // Internal transport entrypoint for the typed Python SDK status operation. #[doc(hidden)] pub fn runtime_status_environment( environment: &EnvironmentHandle, agent: &str, runtime_id: &str, ) -> Result { if environment.provider != "openshell" || environment.control_location != ControlLocation::InEnvControl || [agent, runtime_id] .iter() .any(|value| value.trim().is_empty() || value.contains('\0')) { return Err(FabricError::InvalidConfig { field: "runtime_status".to_string(), reason: "a valid runtime target and OpenShell environment are required".to_string(), }); } let command = crate::task_command::TaskCommand::Status { agent: agent.to_string(), runtime_id: runtime_id.to_string(), }; match crate::environment::command_runtime(environment, &command, 10)? { crate::task_command::TaskCommandResponse::Returned { value } => Ok(value), crate::task_command::TaskCommandResponse::Rejected { error } => { Err(FabricError::EnvironmentProviderOperation { provider: "openshell".to_string(), operation: "status".to_string(), code: if error.effects == "none" { error.code } else { "invalid_response".to_string() }, message: "The runtime status request was rejected.".to_string(), }) } } } static NEXT_ID: AtomicU64 = AtomicU64::new(1); const ADAPTER_PYTHON_ENV: &str = "ADAPTER_PYTHON"; const VIRTUAL_ENV_ENV: &str = "VIRTUAL_ENV"; const LOCAL_HOST_START_TIMEOUT: Duration = Duration::from_secs(90); // Protocol liveness backstop. Adapters remain responsible for normal request // timeouts and should return a normalized response before this bound. const LOCAL_HOST_INVOKE_TIMEOUT: Duration = Duration::from_secs(60 * 60); // Adapters receive a deadline this much earlier than the host timeout so they // can interrupt native work and return a normalized result before it fires. const LOCAL_HOST_ADAPTER_RESERVE: Duration = Duration::from_secs(10); const LOCAL_HOST_STOP_TIMEOUT: Duration = Duration::from_secs(10); const LOCAL_HOST_EXIT_GRACE: Duration = Duration::from_secs(2); const LOCAL_HOST_DIAGNOSTIC_LIMIT: usize = 16 * 1024; #[cfg(test)] pub(crate) const OPENAI_STREAM_HOST: &str = "127.0.0.1"; #[cfg(test)] pub(crate) const OPENAI_STREAM_PROTOCOL_VERSION: &str = "fabric.openai_stream/v1alpha1"; #[cfg(test)] pub(crate) const OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE: &str = "openai.chat_completions.chunk/v1"; #[cfg(not(windows))] const VENV_BIN_DIR: &str = "bin"; #[cfg(windows)] const VENV_BIN_DIR: &str = "Scripts"; #[cfg(not(windows))] const VENV_PYTHON: &str = "python"; #[cfg(windows)] const VENV_PYTHON: &str = "python.exe"; #[cfg(not(windows))] const DEFAULT_PYTHON: &str = "python3"; // `find_on_path` joins each PATH entry with this bare name and stats the result, // so on Windows the fallback must name `python.exe` to be found on disk. #[cfg(windows)] const DEFAULT_PYTHON: &str = "python.exe"; #[cfg(test)] static TEST_STOPPED_AGENTS: Mutex> = Mutex::new(Vec::new()); static LOCAL_HOSTS: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); // Adapter host processes by runtime id. A host's own mutex stays locked for a // whole invocation, so a resource snapshot reads the PID from here instead. static LOCAL_HOST_PROCESSES: LazyLock>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); #[cfg(windows)] static WINDOWS_HOST_CONTROLS: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); static LOCAL_SERVICES: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); /// A request passed to a NeMo Fabric-managed harness runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema, Default)] pub struct RunRequest { /// Request id. pub request_id: String, /// Optional UUID propagation root shared across conversation turns. #[serde(default, skip_serializing_if = "Option::is_none")] pub relay_session_root: Option, /// Request payload for the harness. #[serde(default)] pub input: Value, /// Runtime context such as task, rollout, workflow, or caller metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub context: BTreeMap, /// Per-invocation overrides allowed by the resolved config. #[serde(default, skip_serializing_if = "Option::is_none")] pub overrides: Option, } impl RunRequest { /// Build a text request. pub fn text(input: impl Into) -> Self { Self { request_id: new_id("request"), relay_session_root: None, input: Value::String(input.into()), context: BTreeMap::new(), overrides: None, } } } /// Result from a NeMo Fabric-managed harness invocation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RunResult { /// Stable agent name. pub agent_name: String, /// Stable machine-readable harness identifier used for this run. pub harness: String, /// Adapter used for this run. pub adapter_kind: AdapterKind, /// Adapter implementation id when an adapter descriptor was resolved. #[serde(default, skip_serializing_if = "Option::is_none")] pub adapter_id: Option, /// Runtime handle id. pub runtime_id: String, /// Invocation handle id. pub invocation_id: String, /// Request id. pub request_id: String, /// Runtime status. pub status: RunStatus, /// Primary output. #[serde(default)] pub output: Value, /// Error metadata when applicable. #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, /// Normalized invocation usage when reported by the adapter target. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option, /// Selected software evidence. Absent in older results and when collection is disabled. #[serde( default, skip_serializing_if = "Option::is_none", deserialize_with = "crate::provenance::deserialize_optional_report" )] pub execution: Option, /// Resource snapshot of the runtime, read once after the adapter returned this /// result. Absent when the runtime's resources could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] pub resource_snapshot: Option, /// Invocation duration and endpoint observations from the executing runner. /// Absent in older results and when an execution backend does not collect them. #[serde(default, skip_serializing_if = "Option::is_none")] pub system_stats: Option, /// Artifacts produced or collected by this run. #[serde(default)] pub artifacts: ArtifactManifest, /// Ordered telemetry references when available. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub telemetry: Vec, /// NeMo Fabric lifecycle/progress events emitted during the run. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub events: Vec, /// Adapter-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Runtime completion status. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum RunStatus { /// The harness completed successfully. Succeeded, /// The harness completed with a non-zero status. Failed, /// The invocation or runtime was cancelled. Cancelled, } /// Normalized error metadata. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ErrorInfo { /// NeMo Fabric lifecycle stage where the failure surfaced. pub stage: ErrorStage, /// Stable error code. pub code: String, /// Human-readable error message. pub message: String, /// Whether NeMo Fabric considers this failure safe for a consumer-level retry. #[serde(default)] pub retryable: bool, /// Adapter or runtime metadata useful for diagnostics. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Normalized invocation usage exposed to NeMo Fabric consumers. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RunUsage { /// Input tokens consumed by the invocation. #[serde(default, skip_serializing_if = "Option::is_none")] pub input_tokens: Option, /// Cached input tokens consumed by the invocation, when reported. #[serde(default, skip_serializing_if = "Option::is_none")] pub cached_input_tokens: Option, /// Whether input_tokens already includes cached_input_tokens; absent means unknown. #[serde(default, skip_serializing_if = "Option::is_none")] pub input_tokens_include_cache: Option, /// Output tokens produced by the invocation. #[serde(default, skip_serializing_if = "Option::is_none")] pub output_tokens: Option, /// Total tokens reported by the provider. #[serde(default, skip_serializing_if = "Option::is_none")] pub total_tokens: Option, /// Invocation cost in US dollars when reported by the provider. #[serde(default, skip_serializing_if = "Option::is_none")] #[schemars(range(min = 0.0))] pub cost_usd: Option, /// Adapter-owned usage metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// NeMo Fabric lifecycle stage associated with an error. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ErrorStage { /// Configuration resolution failed. Config, /// Effective config planning failed. Plan, /// Environment preparation failed. Prepare, /// Runtime start/connect failed. Start, /// Runtime invocation failed. Invoke, /// Runtime stop/detach failed. Stop, /// Environment release failed. Release, /// Artifact collection or writing failed. Artifact, } /// Manifest of run artifacts. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ArtifactManifest { /// Artifact root directory. #[serde(default, skip_serializing_if = "Option::is_none")] pub root: Option, /// Artifact entries. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub artifacts: Vec, } /// Reference to one artifact. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ArtifactRef { /// Logical artifact name. pub name: String, /// Artifact kind. pub kind: String, /// Artifact path. pub path: PathBuf, /// Optional media type. #[serde(default, skip_serializing_if = "Option::is_none")] pub media_type: Option, /// Artifact-specific metadata preserved across the Rust and Python SDK boundary. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Reference to telemetry emitted by Relay or another configured telemetry path. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct TelemetryRef { /// Whether Relay was enabled for this run; retained for native compatibility. #[serde(default)] pub relay_enabled: bool, /// Provider in the public SDK reference representation. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider: Option, /// Reference category in the public SDK representation. #[serde(default, skip_serializing_if = "Option::is_none")] pub kind: Option, /// Location of the persisted telemetry. #[serde(default, skip_serializing_if = "Option::is_none")] pub uri: Option, /// Provider trace identity. #[serde(default, skip_serializing_if = "Option::is_none")] pub trace_id: Option, /// Telemetry metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// NeMo Fabric lifecycle or progress event. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct FabricEvent { /// Event id. pub event_id: String, /// Unix timestamp in milliseconds. pub timestamp_millis: u128, /// Event kind. pub kind: String, /// Event message. pub message: String, /// Event metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Provider-specific reference to an existing execution environment. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct EnvironmentReference { /// Environment provider that owns the referenced resource. pub provider: String, /// Provider-specific resource identity used to verify and attach. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub resource: BTreeMap, } /// Resolved execution environment context. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct EnvironmentHandle { /// Environment handle id. pub environment_id: String, /// Environment provider. pub provider: String, /// Where NeMo Fabric control code runs. pub control_location: ControlLocation, /// Workspace visible to the harness runtime. For non-local providers this is /// an environment-local path and must not be resolved on the host. #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace: Option, /// Artifact root visible to the harness runtime. For non-local providers this /// is an environment-local path and must not be resolved on the host. #[serde(default, skip_serializing_if = "Option::is_none")] pub artifacts: Option, /// Environment variables visible to the harness and its tools. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub env: BTreeMap, /// Whether NeMo Fabric owns the environment resource. pub ownership: EnvironmentOwnership, /// Provider connection metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub connection: BTreeMap, /// Provider-specific metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Active or resumable harness runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct RuntimeHandle { /// Runtime handle id. pub runtime_id: String, /// NeMo Fabric-owned opaque binding for this runtime handle. pub runtime_binding: String, /// Agent name. pub agent_name: String, /// Stable machine-readable harness identifier. pub harness: String, /// Adapter kind. pub adapter_kind: AdapterKind, /// Adapter implementation id. #[serde(default, skip_serializing_if = "Option::is_none")] pub adapter_id: Option, /// Service used by this runtime, when it is connected to a shared service. #[serde(default, skip_serializing_if = "Option::is_none")] pub service_id: Option, /// Prepared environment. pub environment: EnvironmentHandle, } /// Ownership of a service exposed through NeMo Fabric. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum ServiceOwnership { /// NeMo Fabric created and owns the service lifecycle. FabricOwned, /// The caller owns the service and NeMo Fabric only owns its attachment. CallerOwned, } /// Caller-supplied reference to an already-running service. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct ServiceReference { /// Adapter that understands and connects to the service. pub adapter_id: String, /// Stable service type understood by the selected adapter. pub service_type: String, /// Non-secret connection fields and credential references. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub connection: BTreeMap, /// Adapter-specific caller-owned service metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// Public identity and binding for one prepared or attached service. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ServiceHandle { /// Unique identifier for this process-local service lifecycle. pub service_id: String, /// NeMo Fabric-owned opaque binding for this service handle. pub service_binding: String, /// Adapter that created or attached to the service. pub adapter_id: String, /// Stable service type returned by the adapter. pub service_type: String, /// Whether NeMo Fabric owns the underlying service. pub ownership: ServiceOwnership, /// Sanitized connection summary. This never contains credential values. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub connection: BTreeMap, /// Adapter-reported version and readiness metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// One request sent to a runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct InvocationHandle { /// Invocation handle id. pub invocation_id: String, /// Request id. pub request_id: String, /// Runtime id. pub runtime_id: String, } /// Context generated for one invocation of a started runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeContext { /// Runtime handle id. pub runtime_id: String, /// Invocation handle id. pub invocation_id: String, /// Request id. pub request_id: String, /// Prepared execution environment. pub environment: EnvironmentHandle, /// Artifact manifest visible to the adapter at invocation start. pub artifacts: ArtifactManifest, /// Runtime telemetry context generated for this invocation. #[serde(default, skip_serializing_if = "Option::is_none")] pub telemetry: Option, /// Unix time in milliseconds by which the adapter should return its result. /// Present on every invocation, shortly before Fabric's own invocation timeout. #[serde(default, skip_serializing_if = "Option::is_none")] pub deadline_millis: Option, /// Absolute folder for the adapter's native state, present when /// `runtime.keep_state` is set. Every runtime of the agent gets the same /// folder, and Fabric lets only one running runtime use it at a time. #[serde(default, skip_serializing_if = "Option::is_none")] pub state_dir: Option, } /// Runtime telemetry config passed to adapters. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeTelemetryContext { /// Whether Relay is enabled for this invocation. pub relay_enabled: bool, /// Generated Relay config path for this invocation. #[serde(default, skip_serializing_if = "Option::is_none")] pub config_path: Option, /// Environment variables NeMo Fabric applies while invoking the adapter. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub env: BTreeMap, /// Additional telemetry metadata surfaced to consumers and adapters. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } /// One invocation against an initialized adapter runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct AdapterInvocation { /// Invocation context generated by NeMo Fabric. pub runtime_context: RuntimeContext, /// Typed caller request for this invocation. pub request: AgentRunRequest, } /// SDK-owned loopback transport for one native OpenAI streaming invocation. #[derive(Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct OpenAiStreamTransport { /// Loopback TCP port owned by the SDK listener. #[schemars(range(min = 1))] pub port: u16, /// Single-use bearer token used to authenticate the adapter connection. #[schemars(length(min = 1), regex(pattern = r"^[^\r\n]*\S[^\r\n]*$"))] pub token: String, } impl std::fmt::Debug for OpenAiStreamTransport { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("OpenAiStreamTransport") .field("port", &self.port) .field("token", &"[REDACTED]") .finish() } } /// Supported southbound native-streaming protocol version. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum OpenAiStreamProtocolVersion { /// Initial authenticated loopback HTTP and chunked-NDJSON protocol. #[serde(rename = "fabric.openai_stream/v1alpha1")] V1Alpha1, } /// Supported OpenAI-compatible chunk profile. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum OpenAiStreamProfile { /// OpenAI Chat Completions chunk objects. #[serde(rename = "openai.chat_completions.chunk/v1")] ChatCompletionsChunkV1, } /// Supported native-streaming listener host. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum OpenAiStreamHost { /// SDK-owned IPv4 loopback listener. #[serde(rename = "127.0.0.1")] Ipv4Loopback, } /// Adapter-facing stream sink with invocation identity generated by NVIDIA NeMo Fabric. #[derive(Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct OpenAiStreamSink { /// Southbound stream protocol version. pub protocol_version: OpenAiStreamProtocolVersion, /// OpenAI event profile emitted on this stream. pub profile: OpenAiStreamProfile, /// Loopback host owned by the SDK listener. pub host: OpenAiStreamHost, /// Loopback TCP port owned by the SDK listener. #[schemars(range(min = 1))] pub port: u16, /// Single-use bearer token. Adapters must not log or persist this value. #[schemars(length(min = 1), regex(pattern = r"^[^\r\n]*\S[^\r\n]*$"))] pub token: String, /// Runtime id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] pub runtime_id: String, /// Invocation id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] pub invocation_id: String, /// Request id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] pub request_id: String, } impl std::fmt::Debug for OpenAiStreamSink { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("OpenAiStreamSink") .field("protocol_version", &self.protocol_version) .field("profile", &self.profile) .field("host", &self.host) .field("port", &self.port) .field("token", &"[REDACTED]") .field("runtime_id", &self.runtime_id) .field("invocation_id", &self.invocation_id) .field("request_id", &self.request_id) .finish() } } /// One adapter-native OpenAI streaming invocation. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct OpenAiStreamInvocation { /// Invocation context generated by NeMo Fabric. pub runtime_context: RuntimeContext, /// Typed caller request for this invocation. pub request: AgentRunRequest, /// Authenticated progressive-output sink for this invocation. pub stream: OpenAiStreamSink, } /// Exact OpenAI object discriminator accepted by the v1 chunk profile. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub enum OpenAiChatCompletionChunkObject { /// OpenAI Chat Completions streaming chunk. #[serde(rename = "chat.completion.chunk")] ChatCompletionChunk, } /// Incremental assistant message fields carried by one OpenAI choice. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct OpenAiChatCompletionChunkDelta { /// Incremental text content. #[serde(default, skip_serializing_if = "Option::is_none")] pub content: Option, /// Incremental refusal content. #[serde(default, skip_serializing_if = "Option::is_none")] pub refusal: Option, /// Incremental message role. #[serde(default, skip_serializing_if = "Option::is_none")] pub role: Option, /// Legacy incremental function-call fields. #[serde(default, skip_serializing_if = "Option::is_none")] pub function_call: Option>, /// Incremental tool-call fields. #[serde(default, skip_serializing_if = "Option::is_none")] pub tool_calls: Option>>, /// Additional OpenAI-compatible delta fields preserved by pass-through. #[serde(flatten)] pub extensions: BTreeMap, } /// One choice within an OpenAI Chat Completions streaming chunk. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct OpenAiChatCompletionChunkChoice { /// Choice index within the response. #[schemars(range(max = u32::MAX))] pub index: u32, /// Incremental assistant message fields. pub delta: OpenAiChatCompletionChunkDelta, /// Terminal reason when this choice finishes. #[serde(default, skip_serializing_if = "Option::is_none")] pub finish_reason: Option, /// Incremental log-probability information. #[serde(default, skip_serializing_if = "Option::is_none")] pub logprobs: Option>, /// Additional OpenAI-compatible choice fields preserved by pass-through. #[serde(flatten)] pub extensions: BTreeMap, } /// OpenAI Chat Completions chunk accepted by the native streaming profile. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct OpenAiChatCompletionChunk { /// Provider-generated response identifier. #[schemars(length(min = 1), regex(pattern = r"\S"))] pub id: String, /// Exact OpenAI streaming object discriminator. pub object: OpenAiChatCompletionChunkObject, /// Unix timestamp in seconds. #[schemars(range(max = u64::MAX))] pub created: u64, /// Model identifier. #[schemars(length(min = 1), regex(pattern = r"\S"))] pub model: String, /// Incremental choices; an explicit usage-only chunk can contain none. pub choices: Vec, /// Optional token-usage data. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage: Option>, /// Additional OpenAI-compatible top-level fields preserved by pass-through. #[serde(flatten)] pub extensions: BTreeMap, } /// One correlated NDJSON record on the adapter-native stream channel. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] pub enum OpenAiStreamRecord { /// One OpenAI Chat Completions chunk. Chunk { /// Monotonic zero-based record sequence. #[schemars(range(max = u64::MAX))] sequence: u64, /// Runtime id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] runtime_id: String, /// Invocation id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] invocation_id: String, /// Request id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] request_id: String, /// OpenAI-compatible chunk passed through to the consumer. chunk: OpenAiChatCompletionChunk, }, /// Explicit successful end of the progressive event channel. End { /// Monotonic zero-based record sequence. #[schemars(range(max = u64::MAX))] sequence: u64, /// Runtime id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] runtime_id: String, /// Invocation id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] invocation_id: String, /// Request id for stream correlation. #[schemars(length(min = 1), regex(pattern = r"\S"))] request_id: String, }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] enum AdapterLifecycleOperation { Start, Invoke, InvokeOpenaiStream, Stop, } impl AdapterLifecycleOperation { fn as_str(self) -> &'static str { match self { Self::Start => "start", Self::Invoke => "invoke", Self::InvokeOpenaiStream => "invoke_openai_stream", Self::Stop => "stop", } } fn error_stage(self) -> ErrorStage { match self { Self::Start => ErrorStage::Start, Self::Invoke | Self::InvokeOpenaiStream => ErrorStage::Invoke, Self::Stop => ErrorStage::Stop, } } } #[derive(Debug, Clone, PartialEq, Serialize)] struct AdapterLifecycleStart { provenance: crate::ProvenanceConfig, agent_name: String, base_dir: PathBuf, config: AgentConfig, runtime_context: RuntimeContext, capability_plan: CapabilityPlan, #[serde(skip_serializing_if = "Option::is_none")] telemetry_plan: Option, #[serde(skip_serializing_if = "Option::is_none")] service: Option, } #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(tag = "operation", rename_all = "snake_case")] enum AdapterServiceContext { Prepare { connection_path: PathBuf, }, Attach { connection_path: PathBuf, reference: ServiceReference, }, Connect { #[serde(skip_serializing_if = "Option::is_none")] execution: Option, service_id: String, connection_path: PathBuf, }, } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] struct AdapterServiceInfo { adapter_id: String, service_type: String, #[serde(default)] connection: BTreeMap, #[serde(default)] metadata: BTreeMap, } #[derive(Debug, Clone, PartialEq, Serialize)] struct AdapterLifecycleStop { runtime_id: String, } #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(tag = "operation", content = "payload", rename_all = "snake_case")] enum AdapterLifecycleRequestKind { Start(Box), Invoke(Box), InvokeOpenaiStream(Box), Stop(AdapterLifecycleStop), } impl AdapterLifecycleRequestKind { fn operation(&self) -> AdapterLifecycleOperation { match self { Self::Start(_) => AdapterLifecycleOperation::Start, Self::Invoke(_) => AdapterLifecycleOperation::Invoke, Self::InvokeOpenaiStream(_) => AdapterLifecycleOperation::InvokeOpenaiStream, Self::Stop(_) => AdapterLifecycleOperation::Stop, } } } #[derive(Debug, Clone, PartialEq, Serialize)] struct AdapterLifecycleRequest { #[serde(flatten)] request: AdapterLifecycleRequestKind, } impl AdapterLifecycleRequest { fn new(request: AdapterLifecycleRequestKind) -> Self { Self { request } } fn operation(&self) -> AdapterLifecycleOperation { self.request.operation() } } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(tag = "status", rename_all = "snake_case")] enum AdapterLifecycleOutcome { Succeeded { #[serde(default)] output: Value, }, Failed { error: ErrorInfo, }, } #[derive(Debug, Clone, PartialEq, Deserialize)] struct AdapterLifecycleResponse { #[serde(default)] execution: Option, operation: AdapterLifecycleOperation, outcome: AdapterLifecycleOutcome, } trait RuntimeAdapter { fn start(&self, plan: &RunPlan, environment: EnvironmentHandle) -> Result; fn invoke( &self, plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, ) -> Result; fn invoke_openai_stream( &self, plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, transport: OpenAiStreamTransport, ) -> Result; fn stop(&self, runtime: &RuntimeHandle) -> Result>; } struct LocalHostAdapter; #[derive(Debug, Clone)] struct RelayRuntimeConfig { path: PathBuf, env: BTreeMap, } struct LocalAdapterHost { service_execution: Option, execution: Option, adapter_execution: Option, child: Child, pending_stop: Vec, #[cfg(windows)] job: Option, stdin: ChildStdin, responses: Receiver>, command: String, runtime_dir: PathBuf, stderr_path: PathBuf, stderr_offset: usize, artifacts: ArtifactManifest, relay_config: Option, // Adapter startup values stay in memory, never in a serialized handle or plan. environment_env: BTreeMap, // Each host holds a descriptor, including runtimes sharing a service's lock. _state_lock: Option, } struct LocalServiceRecord { software_report: Option, host: LocalAdapterHost, runtime: RuntimeHandle, handle: ServiceHandle, connection_path: PathBuf, active_runtime_ids: BTreeSet, state: LocalServiceState, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LocalServiceState { Active, Releasing, ReleaseFailed, Released, } /// Invoke a NeMo Fabric run plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; let mut result = match invoke_runtime(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); return Err(error); } }; match stop_runtime(plan, &runtime) { Ok(events) => result.events.extend(events), Err(error) if result.status == RunStatus::Succeeded => { result.status = RunStatus::Failed; result.error = Some(ErrorInfo { stage: ErrorStage::Stop, code: "runtime_stop_failed".to_string(), message: error.to_string(), retryable: false, metadata: BTreeMap::new(), }); } Err(_) => {} } Ok(result) } /// Prepare an execution environment for a run plan. /// /// This block implements local environment preparation. Other providers return /// [`FabricError::UnsupportedEnvironmentProvider`] until their provider block /// is added. pub fn prepare_environment(plan: &RunPlan) -> Result { let provider_id = planned_environment_provider(plan); let provider = resolve_environment_provider(provider_id).ok_or_else(|| { FabricError::UnsupportedEnvironmentProvider { provider: provider_id.to_string(), adapter_kind: adapter_kind(plan), } })?; provider.prepare(plan) } /// Verify and attach to an existing caller-owned environment. pub fn attach_environment( plan: &RunPlan, reference: &EnvironmentReference, ) -> Result { let provider = planned_environment_provider(plan); if reference.provider != provider { return Err(FabricError::InvalidConfig { field: "environment_reference.provider".to_string(), reason: "reference provider must match the configured environment".to_string(), }); } let provider = resolve_environment_provider(provider).ok_or_else(|| { FabricError::UnsupportedEnvironmentProvider { provider: reference.provider.clone(), adapter_kind: adapter_kind(plan), } })?; provider.attach(plan, reference) } /// Release or detach an environment through its provider. /// /// Caller-owned resources detach without deletion; local files remain in place. pub fn release_environment(environment: &EnvironmentHandle) -> Result<()> { crate::check::invalidate_environment(environment); if environment.provider == "openshell" { return openshell::release(environment); } crate::environment::release_environment(environment).map(|_| ()) } /// Start a runtime in an explicitly prepared environment. /// /// Stopping the runtime does not release the environment. pub fn start_runtime_in(plan: &RunPlan, environment: &EnvironmentHandle) -> Result { validate_start_plan(plan)?; validate_environment_handle(plan, environment)?; start_runtime_in_validated(plan, environment) } /// Prepare a Fabric-owned long-lived service for a resolved run plan. pub fn prepare_service(plan: &RunPlan) -> Result { start_service(plan, ServiceOwnership::FabricOwned, None) } /// Attach to a caller-owned long-lived service for a resolved run plan. pub fn attach_service(plan: &RunPlan, reference: ServiceReference) -> Result { start_service(plan, ServiceOwnership::CallerOwned, Some(reference)) } /// Start a runtime connected to an active service. pub fn start_runtime_with_service( plan: &RunPlan, service: &ServiceHandle, ) -> Result { validate_start_plan(plan)?; validate_service_handle(plan, service)?; let record = local_services() .get(&service.service_id) .cloned() .ok_or_else(|| FabricError::ServiceNotActive { service_id: service.service_id.clone(), })?; let mut record = record.lock().unwrap_or_else(|error| error.into_inner()); if record.handle != *service { return Err(FabricError::ServiceHandleMismatch { field: "handle", expected: record.handle.service_binding.clone(), actual: service.service_binding.clone(), service_id: service.service_id.clone(), }); } if record.state != LocalServiceState::Active { return Err(FabricError::ServiceNotActive { service_id: service.service_id.clone(), }); } // The service host holds the agent's state lock for its connected // runtimes, so only one of them may run at a time. if !record.active_runtime_ids.is_empty() && let Some(path) = agent_state_dir(plan, &record.host.artifacts)? { return Err(FabricError::StateInUse { path }); } let mut current_env = record.host.environment_env.clone(); resolve_named_environment(plan, &mut current_env)?; validate_named_environment(plan, &record.host.environment_env, ¤t_env)?; let state_lock = record .host ._state_lock .as_ref() .map(File::try_clone) .transpose() .map_err(|source| FabricError::Write { path: record.host.runtime_dir.clone(), source, })?; let environment = prepare_environment(plan)?; let runtime = start_local_runtime( plan, environment, Some(( service.service_id.clone(), record.connection_path.clone(), if plan.config.provenance.enabled { record.software_report.clone() } else { None }, state_lock, )), )?; if plan.config.provenance.enabled { let host = { local_hosts().get(&runtime.runtime_id).cloned() }; if let Some(host) = host { let host = host.lock().unwrap_or_else(|error| error.into_inner()); if let Some(report) = host.adapter_execution.as_ref().and_then(service_evidence) { record.software_report = Some(retain_service_snapshot( report, record.software_report.as_ref(), )); } } } record.active_runtime_ids.insert(runtime.runtime_id.clone()); Ok(runtime) } /// Release a Fabric-owned service or detach from a caller-owned service. pub fn release_service(plan: &RunPlan, service: &ServiceHandle) -> Result> { validate_service_handle(plan, service)?; let Some(record) = local_services().get(&service.service_id).cloned() else { return Ok(vec![service_release_event(service, true)]); }; let mut service_record = record.lock().unwrap_or_else(|error| error.into_inner()); if service_record.state == LocalServiceState::Released { return Ok(vec![service_release_event(service, true)]); } if !service_record.active_runtime_ids.is_empty() { return Err(FabricError::ServiceInUse { service_id: service.service_id.clone(), runtime_ids: service_record.active_runtime_ids.iter().cloned().collect(), }); } crate::check::invalidate_service(&service_record.handle); service_record.state = LocalServiceState::Releasing; if let Err(error) = stop_service_host(&mut service_record) { service_record.state = LocalServiceState::ReleaseFailed; return Err(error); } service_record.state = LocalServiceState::Released; drop(service_record); local_services().remove(&service.service_id); Ok(vec![service_release_event(service, false)]) } fn start_service( plan: &RunPlan, ownership: ServiceOwnership, reference: Option, ) -> Result { validate_start_plan(plan)?; if !plan.capabilities.service { return Err(FabricError::UnsupportedRuntimeCapability { adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), capability: "service", }); } let environment = prepare_environment(plan)?; validate_local_host(plan, &environment)?; let service_id = new_id("service"); let runtime_binding = runtime_binding(&service_id, None, plan, &environment)?; let runtime = RuntimeHandle { runtime_id: service_id.clone(), runtime_binding, agent_name: plan.agent_name.clone(), harness: harness(plan), adapter_kind: adapter_kind(plan), adapter_id: adapter_id(plan), service_id: None, environment, }; let invocation = InvocationHandle { invocation_id: new_id("service-start"), request_id: new_id("service-start-request"), runtime_id: service_id.clone(), }; let mut artifacts = artifact_manifest(plan)?; let fabric_home = prepare_fabric_home(&artifacts, &runtime, &invocation)?; let relay_config = prepare_relay_runtime_config(plan, &runtime, &fabric_home, &mut artifacts)?; let connection_path = local_host_directory(&service_id).join("service-connection.json"); let service_context = match reference { Some(reference) => AdapterServiceContext::Attach { connection_path: connection_path.clone(), reference, }, None => AdapterServiceContext::Prepare { connection_path: connection_path.clone(), }, }; let start = adapter_lifecycle_start( plan, &runtime, &invocation, &artifacts, relay_config.as_ref(), Some(service_context), )?; let environment_env = start.runtime_context.environment.env.clone(); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); let mut host = spawn_local_host( plan, &runtime, artifacts, relay_config, environment_env, None, )?; let output = match exchange_lifecycle_message( &mut host, &service_id, &request, LOCAL_HOST_START_TIMEOUT, ) { Ok(output) => output, Err(error) => { let _ = terminate_local_host(&mut host); let _ = remove_local_host_files(&host); return Err(error); } }; let info: AdapterServiceInfo = serde_json::from_value(output).map_err(|source| { let _ = terminate_local_host(&mut host); let _ = remove_local_host_files(&host); lifecycle_error( AdapterLifecycleOperation::Start, &service_id, "invalid_service_info", format!("adapter returned invalid service information: {source}"), local_host_diagnostics(&host), ) })?; let software_report = if plan.config.provenance.enabled { host.adapter_execution .as_ref() .and_then(service_evidence) .or_else(|| service_metadata_evidence(&info)) } else { None }; let service_binding = service_binding(&service_id, plan, ownership, &info)?; let handle = ServiceHandle { service_id: service_id.clone(), service_binding, adapter_id: info.adapter_id, service_type: info.service_type, ownership, connection: info.connection, metadata: info.metadata, }; let levels = plan .capabilities .metadata .get("service_health_checks") .and_then(Value::as_array) .map(|levels| { levels .iter() .filter_map(|level| serde_json::from_value(level.clone()).ok()) .collect() }) .unwrap_or_default(); crate::check::register_service(&handle, &runtime, levels); local_services().insert( service_id, Arc::new(Mutex::new(LocalServiceRecord { software_report, host, runtime, handle: handle.clone(), connection_path, active_runtime_ids: BTreeSet::new(), state: LocalServiceState::Active, })), ); Ok(handle) } fn validate_start_plan(plan: &RunPlan) -> Result<()> { validate_config(&plan.config)?; validate_agent_config(&plan.agent_config)?; validate_harness_settings(&plan.config, plan.adapter_descriptor.as_ref())?; validate_workflow(&plan.config, plan.adapter_target_descriptor.as_ref())?; validate_adapter_compatibility(plan)?; validate_tool_definitions(&plan.config, plan.adapter_descriptor.as_ref())?; validate_agent_config_extensions( &plan.config, plan.adapter_descriptor.as_ref(), plan.adapter_target_descriptor.as_ref(), ) } /// Start or connect to a harness runtime in its configured local environment. /// /// For a non-local provider, prepare or attach to the environment explicitly /// and call [`start_runtime_in`] with the returned handle. pub fn start_runtime(plan: &RunPlan) -> Result { validate_start_plan(plan)?; let provider = planned_environment_provider(plan); if provider != "local" { return Err(FabricError::EnvironmentHandleRequired { provider: provider.to_string(), }); } let environment = prepare_environment(plan)?; start_runtime_in_validated(plan, &environment) } /// Invoke a started harness runtime. pub fn invoke_runtime( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, ) -> Result { validate_adapter_compatibility(plan)?; validate_runtime_handle(plan, runtime)?; if uses_local_host(plan) { if runtime.environment.provider == "openshell" { return OpenShellRuntimeAdapter.invoke(plan, runtime, request); } return LocalHostAdapter.invoke(plan, runtime, request); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind: adapter_kind(plan), }) } /// Read a timestamped resource snapshot of a started runtime. /// /// The snapshot covers the runtime's local adapter host process and its live /// descendants. It does not wait for a running invocation. It returns /// [`FabricError::ResourceObservationUnavailable`] when the runtime stopped, is /// stopping, its host exited, a live process in its tree or the process table /// cannot be read, or the process tree kept changing during the read, and /// [`FabricError::ResourceObservationUnsupported`] when the adapter kind, /// environment provider, or operating system has no resource reader. pub fn runtime_resource_snapshot( plan: &RunPlan, runtime: &RuntimeHandle, ) -> Result { validate_runtime_handle(plan, runtime)?; if !uses_local_host(plan) { return Err(FabricError::ResourceObservationUnsupported { runtime_id: runtime.runtime_id.clone(), reason: format!( "adapter kind `{}` has no resource reader", adapter_kind_name(runtime.adapter_kind) ), }); } local_runtime_resource_snapshot(runtime) } fn local_runtime_resource_snapshot(runtime: &RuntimeHandle) -> Result { if runtime.environment.provider != "local" { return Err(FabricError::ResourceObservationUnsupported { runtime_id: runtime.runtime_id.clone(), reason: format!( "environment provider `{}` has no resource reader", runtime.environment.provider ), }); } // Clone the entry so the map lock is released before the process tree is read. let host = local_host_processes().get(&runtime.runtime_id).cloned(); snapshot_local_host(&runtime.runtime_id, host.as_ref()) } /// Invoke a started harness runtime and pass through native OpenAI chat-completion chunks. pub fn invoke_openai_stream( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, transport: OpenAiStreamTransport, ) -> Result { validate_adapter_compatibility(plan)?; validate_runtime_handle(plan, runtime)?; let descriptor_supports_streaming = plan .adapter_descriptor .as_ref() .is_some_and(|adapter| adapter.descriptor.capabilities.streaming); if !plan.capabilities.streaming || !descriptor_supports_streaming { return Err(FabricError::UnsupportedRuntimeCapability { adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), capability: "streaming", }); } validate_openai_stream_transport(&transport)?; if uses_local_host(plan) { if runtime.environment.provider == "openshell" { return OpenShellRuntimeAdapter.invoke_openai_stream(plan, runtime, request, transport); } return LocalHostAdapter.invoke_openai_stream(plan, runtime, request, transport); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind: adapter_kind(plan), }) } fn validate_openai_stream_transport(transport: &OpenAiStreamTransport) -> Result<()> { if transport.port == 0 { return Err(FabricError::InvalidOpenAiStreamTransport { field: "port", reason: "must be greater than zero", }); } if transport.token.trim().is_empty() || transport.token.contains('\r') || transport.token.contains('\n') { return Err(FabricError::InvalidOpenAiStreamTransport { field: "token", reason: "must be a non-empty string without line breaks", }); } Ok(()) } fn validate_adapter_compatibility(plan: &RunPlan) -> Result<()> { validate_adapter_config_compatibility( &plan.config, plan.adapter_descriptor .as_ref() .map(|adapter| &adapter.descriptor), )?; if let Some(route) = plan .capability_plan .routes .iter() .find(|route| route.target == CapabilityTarget::Unsupported) { return Err(FabricError::AdapterCompatibility { adapter_id: adapter_id(plan).unwrap_or_else(|| harness(plan)), field: route.config_field(), reason: route.reason.clone(), }); } Ok(()) } /// Stop or detach from a harness runtime. pub fn stop_runtime(plan: &RunPlan, runtime: &RuntimeHandle) -> Result> { validate_runtime_handle(plan, runtime)?; crate::check::invalidate_runtime(runtime); if uses_local_host(plan) { let stopped = if runtime.environment.provider == "openshell" { OpenShellRuntimeAdapter.stop(runtime) } else { LocalHostAdapter.stop(runtime) }; // Release the service reference only after owned host cleanup is confirmed. if !local_hosts().contains_key(&runtime.runtime_id) && let Some(service_id) = runtime.service_id.as_ref() && let Some(record) = local_services().get(service_id).cloned() { record .lock() .unwrap_or_else(|error| error.into_inner()) .active_runtime_ids .remove(&runtime.runtime_id); } return stopped; } Err(FabricError::UnsupportedRuntimeAdapter { harness: runtime.harness.clone(), adapter_kind: runtime.adapter_kind, }) } fn uses_local_host(plan: &RunPlan) -> bool { matches!( adapter_kind(plan), AdapterKind::Process | AdapterKind::Python ) } fn start_runtime_in_validated( plan: &RunPlan, environment: &EnvironmentHandle, ) -> Result { if environment.provider == "openshell" { if uses_local_host(plan) { return OpenShellRuntimeAdapter.start(plan, environment.clone()); } return Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind: adapter_kind(plan), }); } if environment.provider != "local" { return Err(FabricError::UnsupportedEnvironmentProvider { provider: environment.provider.clone(), adapter_kind: adapter_kind(plan), }); } if uses_local_host(plan) { return LocalHostAdapter.start(plan, environment.clone()); } Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind: adapter_kind(plan), }) } fn planned_environment_provider(plan: &RunPlan) -> &str { plan.environment_plan .as_ref() .map_or("local", |environment| environment.provider.as_str()) } fn validate_environment_handle(plan: &RunPlan, environment: &EnvironmentHandle) -> Result<()> { if environment.environment_id.trim().is_empty() { return environment_handle_mismatch(environment, "environment_id", "non-empty", ""); } expect_environment_field( environment, "provider", planned_environment_provider(plan), &environment.provider, )?; let (expected_control, expected_ownership) = plan.environment_plan.as_ref().map_or( ( ControlLocation::ExternalControl, EnvironmentOwnership::CallerOwned, ), |configured| (configured.control_location, configured.ownership), ); expect_environment_field( environment, "control_location", control_location_name(expected_control), control_location_name(environment.control_location), )?; expect_environment_field( environment, "ownership", environment_ownership_name(expected_ownership), environment_ownership_name(environment.ownership), )?; let (expected_workspace, expected_artifacts) = if let Some(configured) = &plan.environment_plan { ( configured .workspace .clone() .map(absolute_path) .transpose()?, configured.artifacts.clone(), ) } else { ( Some(absolute_path(plan.base_dir.clone())?), plan.config .runtime .artifacts .as_ref() .map(|artifacts| resolve_path(&plan.base_dir, artifacts)), ) }; if environment.workspace != expected_workspace { return environment_handle_mismatch( environment, "workspace", &format!("{expected_workspace:?}"), &format!("{:?}", environment.workspace), ); } if environment.artifacts != expected_artifacts { return environment_handle_mismatch( environment, "artifacts", &format!("{expected_artifacts:?}"), &format!("{:?}", environment.artifacts), ); } Ok(()) } fn expect_environment_field( environment: &EnvironmentHandle, field: &'static str, expected: &str, actual: &str, ) -> Result<()> { if expected == actual { return Ok(()); } environment_handle_mismatch(environment, field, expected, actual) } fn environment_handle_mismatch( environment: &EnvironmentHandle, field: &'static str, expected: &str, actual: &str, ) -> Result { Err(FabricError::EnvironmentHandleMismatch { field, expected: expected.to_string(), actual: actual.to_string(), environment_id: environment.environment_id.clone(), }) } fn control_location_name(location: ControlLocation) -> &'static str { match location { ControlLocation::ExternalControl => "external_control", ControlLocation::InEnvControl => "in_env_control", } } fn environment_ownership_name(ownership: EnvironmentOwnership) -> &'static str { match ownership { EnvironmentOwnership::CallerOwned => "caller_owned", EnvironmentOwnership::FabricOwned => "fabric_owned", } } fn validate_runtime_handle(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<()> { let expected_binding = runtime_binding( &runtime.runtime_id, runtime.service_id.as_ref(), plan, &runtime.environment, )?; expect_runtime_field( runtime, "runtime_binding", &expected_binding, &runtime.runtime_binding, )?; expect_runtime_field(runtime, "agent_name", &plan.agent_name, &runtime.agent_name)?; expect_runtime_field(runtime, "harness", &harness(plan), &runtime.harness)?; expect_runtime_field( runtime, "adapter_kind", &adapter_kind_name(adapter_kind(plan)), &adapter_kind_name(runtime.adapter_kind), )?; expect_runtime_field( runtime, "adapter_id", &optional_runtime_value(adapter_id(plan).as_deref()), &optional_runtime_value(runtime.adapter_id.as_deref()), )?; Ok(()) } fn expect_runtime_field( runtime: &RuntimeHandle, field: &'static str, expected: &str, actual: &str, ) -> Result<()> { if expected == actual { return Ok(()); } Err(FabricError::RuntimeHandleMismatch { field, expected: expected.to_string(), actual: actual.to_string(), runtime_id: runtime.runtime_id.clone(), }) } #[derive(Serialize)] struct RuntimeBindingMaterial<'a> { runtime_id: &'a str, service_id: Option<&'a String>, environment_id: &'a str, plan: &'a RunPlan, environment: RuntimeEnvironmentBinding<'a>, } #[derive(Serialize)] struct RuntimeEnvironmentBinding<'a> { provider: &'a str, control_location: ControlLocation, workspace: &'a Option, artifacts: &'a Option, env: &'a BTreeMap, ownership: EnvironmentOwnership, connection: &'a BTreeMap, metadata: &'a BTreeMap, } fn runtime_environment_binding(environment: &EnvironmentHandle) -> RuntimeEnvironmentBinding<'_> { RuntimeEnvironmentBinding { provider: &environment.provider, control_location: environment.control_location, workspace: &environment.workspace, artifacts: &environment.artifacts, env: &environment.env, ownership: environment.ownership, connection: &environment.connection, metadata: &environment.metadata, } } fn runtime_binding( runtime_id: &str, service_id: Option<&String>, plan: &RunPlan, environment: &EnvironmentHandle, ) -> Result { let mut binding_plan = plan.clone(); binding_plan.config.provenance.reports = None; stable_hash( "fabric-runtime-binding", &RuntimeBindingMaterial { runtime_id, service_id, environment_id: &environment.environment_id, plan: &binding_plan, environment: runtime_environment_binding(environment), }, ) } #[derive(Serialize)] struct ServiceBindingMaterial<'a> { service_id: &'a str, plan: &'a RunPlan, ownership: ServiceOwnership, adapter_id: &'a str, service_type: &'a str, connection: &'a BTreeMap, metadata: &'a BTreeMap, } fn service_binding( service_id: &str, plan: &RunPlan, ownership: ServiceOwnership, info: &AdapterServiceInfo, ) -> Result { let mut binding_plan = plan.clone(); // Bind relative plans to the directory used when the service was prepared. binding_plan.base_dir = absolute_path(binding_plan.base_dir)?; binding_plan.config.provenance = Default::default(); stable_hash( "fabric-service-binding", &ServiceBindingMaterial { service_id, plan: &binding_plan, ownership, adapter_id: &info.adapter_id, service_type: &info.service_type, connection: &info.connection, metadata: &info.metadata, }, ) } fn validate_service_handle(plan: &RunPlan, service: &ServiceHandle) -> Result<()> { let info = AdapterServiceInfo { adapter_id: service.adapter_id.clone(), service_type: service.service_type.clone(), connection: service.connection.clone(), metadata: service.metadata.clone(), }; let expected = service_binding(&service.service_id, plan, service.ownership, &info)?; if expected != service.service_binding { return Err(FabricError::ServiceHandleMismatch { field: "service_binding", expected, actual: service.service_binding.clone(), service_id: service.service_id.clone(), }); } if service.adapter_id != adapter_id(plan).unwrap_or_else(|| harness(plan)) { return Err(FabricError::ServiceHandleMismatch { field: "adapter_id", expected: adapter_id(plan).unwrap_or_else(|| harness(plan)), actual: service.adapter_id.clone(), service_id: service.service_id.clone(), }); } Ok(()) } fn stable_hash(prefix: &str, value: &T) -> Result { let bytes = serde_json::to_vec(value).map_err(FabricError::SerializeJson)?; let mut hash = 0xcbf29ce484222325_u64; for byte in bytes { hash ^= u64::from(byte); hash = hash.wrapping_mul(0x100000001b3); } Ok(format!("{prefix}-{hash:016x}")) } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct ProcessAdapterSettings { command: String, #[serde(default)] script: Option, #[serde(default)] args: Vec, #[serde(default)] cwd: Option, #[serde(default)] env: BTreeMap, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct PythonAdapterSettings { module: String, #[serde(default)] python: Option, #[serde(default)] python_env: Option, #[serde(default)] args: Vec, #[serde(default)] cwd: Option, #[serde(default)] env: BTreeMap, } #[derive(Debug, Clone, PartialEq)] struct PythonCommand { path: PathBuf, source: PythonSource, } /// Where a resolved Python interpreter came from. Drives clear preflight error /// messages and how the interpreter is validated (concrete file vs. PATH lookup). #[derive(Debug, Clone, PartialEq)] enum PythonSource { /// `harness.settings.python`. Setting, /// `harness.settings.python_env` (the named environment variable). SettingEnv(String), /// The `ADAPTER_PYTHON` environment variable. AdapterPythonEnv, /// The active virtualenv (`VIRTUAL_ENV`). Virtualenv, /// An interpreter found next to the running NeMo Fabric executable. HostInterpreter, /// The bare `python3` command resolved off `PATH` (last resort). DefaultPython3, } impl PythonSource { fn describe(&self) -> String { match self { PythonSource::Setting => "harness.settings.python".to_string(), PythonSource::SettingEnv(name) => { format!("harness.settings.python_env (`{name}`)") } PythonSource::AdapterPythonEnv => { format!("`{ADAPTER_PYTHON_ENV}` environment variable") } PythonSource::Virtualenv => format!("active virtualenv (`{VIRTUAL_ENV_ENV}`)"), PythonSource::HostInterpreter => "NeMo Fabric host interpreter".to_string(), PythonSource::DefaultPython3 => format!("default `{DEFAULT_PYTHON}` on PATH"), } } } impl RuntimeAdapter for LocalHostAdapter { fn start(&self, plan: &RunPlan, environment: EnvironmentHandle) -> Result { start_local_runtime(plan, environment, None) } fn invoke( &self, plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, ) -> Result { run_local_host_adapter(plan, runtime, request) } fn invoke_openai_stream( &self, plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, transport: OpenAiStreamTransport, ) -> Result { run_local_host_openai_stream_adapter(plan, runtime, request, transport) } fn stop(&self, runtime: &RuntimeHandle) -> Result> { let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { #[cfg(not(windows))] let process = local_host_processes().get(&runtime.runtime_id).cloned(); #[cfg(not(windows))] if process .as_ref() .is_some_and(|process| !owned_process_has_exited(process)) { return Err(cleanup_unconfirmed(runtime)); } let pending = STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) .get(&runtime.runtime_id) .cloned() .unwrap_or_default(); confirm_stopped_descendants(runtime, &pending)?; local_host_processes().remove(&runtime.runtime_id); #[cfg(windows)] windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); return Ok(vec![local_host_stop_event(runtime, true, false)]); }; let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; let mut interrupted = None; let mut guard = loop { match host.try_lock() { Ok(guard) => break guard, Err(std::sync::TryLockError::Poisoned(error)) => break error.into_inner(), Err(std::sync::TryLockError::WouldBlock) => { if interrupted.is_none() { interrupted = Some(interrupt_local_host(runtime)?); } if Instant::now() >= deadline { return Err(cleanup_unconfirmed(runtime)); } thread::sleep(Duration::from_millis(10)); } } }; if let Some(processes) = interrupted { guard.pending_stop.extend(processes); } if let Some(processes) = STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) .get(&runtime.runtime_id) { guard.pending_stop.extend(processes.clone()); } #[cfg(unix)] if guard .child .try_wait() .map_err(|_| cleanup_unconfirmed(runtime))? .is_none() { let pid = guard.child.id(); guard.pending_stop.extend( host_descendants(pid) .into_iter() .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())), ); } let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: runtime.runtime_id.clone(), })); let result = exchange_lifecycle_message( &mut guard, &runtime.runtime_id, &request, LOCAL_HOST_STOP_TIMEOUT, ); terminate_local_host(&mut guard).map_err(|_| cleanup_unconfirmed(runtime))?; confirm_stopped_descendants(runtime, &guard.pending_stop)?; remove_local_host_files(&guard).map_err(|_| cleanup_unconfirmed(runtime))?; local_hosts().remove(&runtime.runtime_id); local_host_processes().remove(&runtime.runtime_id); #[cfg(windows)] windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" ); if !host_crashed { result?; } #[cfg(test)] TEST_STOPPED_AGENTS .lock() .expect("stop tracker") .push(runtime.agent_name.clone()); Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) } } fn cleanup_unconfirmed(runtime: &RuntimeHandle) -> FabricError { lifecycle_error( AdapterLifecycleOperation::Stop, &runtime.runtime_id, "cleanup_unconfirmed", "Owned process cleanup is unconfirmed; retry stop before replacement.", "", ) } fn owned_process_has_exited(process: &LocalHostProcess) -> bool { if process.has_exited() == Some(true) { return true; } #[cfg(unix)] { let Ok(pid) = libc::pid_t::try_from(process.pid()) else { return false; }; if pid <= 0 { return false; } // SAFETY: signal zero only probes existence. An unidentified process is // never signalled; ESRCH can still prove that its PID is now absent. (unsafe { libc::kill(pid, 0) } != 0) && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) } #[cfg(not(unix))] false } #[cfg(unix)] fn signal_owned_process(process: &LocalHostProcess) -> std::io::Result<()> { if owned_process_has_exited(process) { return Ok(()); } if !process.is_current() { return Err(std::io::Error::other("process ownership is unconfirmed")); } // SAFETY: the process identity was checked immediately before signalling. if unsafe { libc::kill(process.pid() as libc::pid_t, libc::SIGKILL) } != 0 { let error = std::io::Error::last_os_error(); if error.raw_os_error() != Some(libc::ESRCH) { return Err(error); } } Ok(()) } fn interrupt_local_host(runtime: &RuntimeHandle) -> Result> { #[cfg(unix)] { let process = local_host_processes() .get(&runtime.runtime_id) .cloned() .ok_or_else(|| cleanup_unconfirmed(runtime))?; if process.has_exited() == Some(true) { return Ok(Vec::new()); } if !process.is_current() { return Err(cleanup_unconfirmed(runtime)); } let descendants: Vec<_> = host_descendants(process.pid()) .into_iter() .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) .collect(); let descendants = { let mut retained = STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()); let previous = retained.entry(runtime.runtime_id.clone()).or_default(); previous.extend(descendants); previous.clone() }; for child in descendants.iter().rev() { signal_owned_process(child).map_err(|_| cleanup_unconfirmed(runtime))?; } signal_owned_process(&process).map_err(|_| cleanup_unconfirmed(runtime))?; Ok(descendants) } #[cfg(windows)] { let control = windows_host_controls() .get(&runtime.runtime_id) .cloned() .ok_or_else(|| cleanup_unconfirmed(runtime))?; control .interrupt() .map_err(|_| cleanup_unconfirmed(runtime))?; Ok(Vec::new()) } #[cfg(not(any(unix, windows)))] Err(cleanup_unconfirmed(runtime)) } fn confirm_stopped_descendants( runtime: &RuntimeHandle, processes: &[LocalHostProcess], ) -> Result<()> { #[cfg(windows)] let control = windows_host_controls().get(&runtime.runtime_id).cloned(); let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; loop { let exited = processes.iter().all(owned_process_has_exited); #[cfg(windows)] let exited = exited && control .as_ref() .is_none_or(|control| control.cleanup_confirmed() == Some(true)); if exited { return Ok(()); } #[cfg(unix)] for process in processes { signal_owned_process(process).map_err(|_| cleanup_unconfirmed(runtime))?; } #[cfg(windows)] if let Some(control) = &control && control.job.is_some() { control .interrupt() .map_err(|_| cleanup_unconfirmed(runtime))?; } if Instant::now() >= deadline { return Err(cleanup_unconfirmed(runtime)); } thread::sleep(Duration::from_millis(10)); } } fn start_local_runtime( plan: &RunPlan, environment: EnvironmentHandle, service: Option<( String, PathBuf, Option, Option, )>, ) -> Result { validate_local_host(plan, &environment)?; let service_id = service .as_ref() .map(|(service_id, _, _, _)| service_id.clone()); let runtime_id = new_id("runtime"); let runtime_binding = runtime_binding(&runtime_id, service_id.as_ref(), plan, &environment)?; let runtime = RuntimeHandle { runtime_id, runtime_binding, agent_name: plan.agent_name.clone(), harness: harness(plan), adapter_kind: adapter_kind(plan), adapter_id: adapter_id(plan), service_id, environment, }; let start_invocation = InvocationHandle { invocation_id: new_id("runtime-start"), request_id: new_id("runtime-start-request"), runtime_id: runtime.runtime_id.clone(), }; let mut artifacts = artifact_manifest(plan)?; let fabric_home = prepare_fabric_home(&artifacts, &runtime, &start_invocation)?; let relay_config = prepare_relay_runtime_config(plan, &runtime, &fabric_home, &mut artifacts)?; let service_execution = service .as_ref() .and_then(|(_, _, report, _)| report.clone()); let (service_context, state_lock) = match service { Some((service_id, connection_path, execution, state_lock)) => ( Some(AdapterServiceContext::Connect { execution, service_id, connection_path, }), state_lock, ), None => (None, None), }; let start = adapter_lifecycle_start( plan, &runtime, &start_invocation, &artifacts, relay_config.as_ref(), service_context, )?; let environment_env = start.runtime_context.environment.env.clone(); let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Start(Box::new(start))); let started_at = SystemTime::now(); let levels = plan .capabilities .metadata .get("health_checks") .and_then(Value::as_array) .map(|levels| { levels .iter() .filter_map(|level| serde_json::from_value(level.clone()).ok()) .collect() }) .unwrap_or_default(); crate::check::register(&runtime, levels); let mut host = match spawn_local_host( plan, &runtime, artifacts, relay_config, environment_env, state_lock, ) { Ok(host) => host, Err(error) => { crate::check::invalidate_runtime(&runtime); return Err(error); } }; host.service_execution = service_execution; if let Err(error) = exchange_lifecycle_message( &mut host, &runtime.runtime_id, &request, LOCAL_HOST_START_TIMEOUT, ) { crate::check::invalidate_runtime(&runtime); let _ = terminate_local_host(&mut host); let _ = remove_local_host_files(&host); return Err(error); } #[cfg(windows)] match WindowsHostControl::capture(&host) { Ok(control) => { windows_host_controls().insert(runtime.runtime_id.clone(), Arc::new(control)); } Err(source) => { crate::check::invalidate_runtime(&runtime); let _ = terminate_local_host(&mut host); let _ = remove_local_host_files(&host); return Err(FabricError::ProcessRunner { command: host.command.clone(), source, }); } } local_host_processes().insert( runtime.runtime_id.clone(), LocalHostProcess::new(host.child.id(), started_at), ); local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); Ok(runtime) } fn validate_local_host(plan: &RunPlan, environment: &EnvironmentHandle) -> Result<()> { if environment.provider != "local" { return Err(FabricError::UnsupportedEnvironmentProvider { provider: environment.provider.clone(), adapter_kind: adapter_kind(plan), }); } if !matches!( adapter_kind(plan), AdapterKind::Process | AdapterKind::Python ) { return Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind: adapter_kind(plan), }); } if adapter_kind(plan) == AdapterKind::Python { preflight_python_adapter(plan)?; } Ok(()) } fn run_local_host_adapter( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, ) -> Result { run_local_host_adapter_with_timeout(plan, runtime, request, local_host_invoke_timeout(plan)?) } fn run_local_host_openai_stream_adapter( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, transport: OpenAiStreamTransport, ) -> Result { run_local_host_invocation_with_timeout( plan, runtime, request, LocalHostInvocation::OpenAiStream(transport), local_host_invoke_timeout(plan)?, ) } fn local_host_invoke_timeout(plan: &RunPlan) -> Result { match plan.config.runtime.timeout_seconds { Some(seconds) if seconds <= 0.0 => Err(FabricError::InvalidConfig { field: "runtime.timeout_seconds".to_string(), reason: "must be a finite number greater than zero".to_string(), }), Some(seconds) => { Duration::try_from_secs_f64(seconds).map_err(|_| FabricError::InvalidConfig { field: "runtime.timeout_seconds".to_string(), reason: "must be a finite number greater than zero".to_string(), }) } None => Ok(LOCAL_HOST_INVOKE_TIMEOUT), } } fn run_local_host_adapter_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, request: RunRequest, invoke_timeout: Duration, ) -> Result { run_local_host_invocation_with_timeout( plan, runtime, request, LocalHostInvocation::Invoke, invoke_timeout, ) } enum LocalHostInvocation { Invoke, OpenAiStream(OpenAiStreamTransport), } impl LocalHostInvocation { fn operation(&self) -> AdapterLifecycleOperation { match self { Self::Invoke => AdapterLifecycleOperation::Invoke, Self::OpenAiStream(_) => AdapterLifecycleOperation::InvokeOpenaiStream, } } } fn run_local_host_invocation_with_timeout( plan: &RunPlan, runtime: &RuntimeHandle, mut request: RunRequest, invocation_kind: LocalHostInvocation, invoke_timeout: Duration, ) -> Result { let operation = invocation_kind.operation(); if request.request_id.is_empty() { request.request_id = new_id("request"); } let invocation = InvocationHandle { invocation_id: new_id("invocation"), request_id: request.request_id.clone(), runtime_id: runtime.runtime_id.clone(), }; let host = local_hosts() .get(&runtime.runtime_id) .cloned() .ok_or_else(|| { lifecycle_error( operation, &runtime.runtime_id, "host_unavailable", "persistent local adapter host is not active", "", ) })?; let exchange_result = { let mut host_guard = host.lock().unwrap_or_else(|error| error.into_inner()); let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; let mut adapter_invocation = adapter_invocation( plan, runtime, &invocation, &request, &artifacts, relay_config.as_ref(), )?; validate_named_environment( plan, &host_guard.environment_env, &adapter_invocation.runtime_context.environment.env, )?; adapter_invocation.runtime_context.deadline_millis = adapter_deadline_millis(invoke_timeout); let (lifecycle_request, adapter_payload) = match invocation_kind { LocalHostInvocation::Invoke => { let mut persisted = adapter_invocation.clone(); redact_adapter_invocation(&mut persisted); let payload = serde_json::to_string_pretty(&persisted).map_err(FabricError::SerializeJson)?; ( AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Invoke(Box::new( adapter_invocation, ))), payload, ) } LocalHostInvocation::OpenAiStream(transport) => { let streaming_invocation = OpenAiStreamInvocation { stream: openai_stream_sink(runtime, &invocation, transport), runtime_context: adapter_invocation.runtime_context, request: adapter_invocation.request, }; let mut persisted = streaming_invocation.clone(); redact_openai_stream_invocation(&mut persisted); let payload = serde_json::to_string_pretty(&persisted).map_err(FabricError::SerializeJson)?; ( AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::InvokeOpenaiStream( Box::new(streaming_invocation), )), payload, ) } }; let fabric_invocation = write_fabric_invocation(&fabric_home, &adapter_payload)?; let stats_config = plan.config.system_stats.as_ref(); let reader = SystemReader { runtime_id: runtime.runtime_id.clone(), host: local_host_processes().get(&runtime.runtime_id).cloned(), local: runtime.environment.provider == "local", collect: stats_config .map(|config| config.collect.clone()) .unwrap_or_default(), }; let before = stats_config.map(|_| reader.read()); let sampling = stats_config .and_then(|config| config.sampling.as_ref()) .zip(before.as_ref()) .map(|(config, before)| { StatsSession::prepare(config, &invocation, reader.clone(), before, &fabric_home) }); let invocation_started = Instant::now(); if let Some(sampling) = &sampling { sampling.start(invocation_started); } let exchange = exchange_lifecycle_message( &mut host_guard, &runtime.runtime_id, &lifecycle_request, invoke_timeout, ); // Close the memory-only sampling window before reading the clock. No // OS collection or sample-file I/O runs under this registration lock. if let Some(sampling) = &sampling { sampling.close(); } let elapsed_seconds = invocation_started.elapsed().as_secs_f64(); // Endpoint collection and final artifact writes are outside elapsed time, // and still under the host lock to exclude the next invocation. let after = stats_config.map(|_| reader.read()); // Keep the independent final resource_snapshot contract even when the // invocation stats block or its process group is disabled. let resource_snapshot = if stats_config .is_some_and(|config| config.includes(crate::SystemStatsMetricGroup::Process)) { after .as_ref() .and_then(|reading| reading.adapter_tree.snapshot()) .cloned() } else { local_runtime_resource_snapshot(runtime).ok() }; let system_stats = before .zip(after) .zip(stats_config) .map(|((before, after), config)| { let mut stats = InvocationSystemStats::new( before.adapter_tree, after.adapter_tree.clone(), elapsed_seconds, ); stats.collected_groups = config.collect.clone(); if config .collect .iter() .any(|group| *group != crate::SystemStatsMetricGroup::Process) { let mut cgroup = CgroupStats::new(before.cgroup, after.cgroup.clone()); if !config.includes(crate::SystemStatsMetricGroup::CgroupCpu) { cgroup.cpu_usage_microseconds = crate::Measurement::NotCollected; cgroup.cpu_throttled_microseconds = crate::Measurement::NotCollected; } if !config.includes(crate::SystemStatsMetricGroup::CgroupMemory) { cgroup.oom_kills = crate::Measurement::NotCollected; } stats.cgroup = Some(cgroup); } if let Some(sampling) = &sampling { stats.sampled = Some(sampling.finish(elapsed_seconds, &after)); } stats }); // Optional evidence never replaces the adapter's original outcome. let stats_summary = system_stats .as_ref() .filter(|_| sampling.is_some()) .map(|stats| crate::system_stats::persist_summary(stats, &invocation, &fabric_home)); match exchange { Ok(output) => { let stderr = take_local_host_stderr(&mut host_guard); Ok(( output, resource_snapshot, system_stats, { let mut execution = host_guard.execution.clone(); if let (Some(report), Some(adapter)) = (&mut execution, &host_guard.adapter_execution) { append_adapter_evidence( report, adapter, host_guard.service_execution.as_ref(), ); } else if let (Some(report), Some(service)) = (&mut execution, &host_guard.service_execution) { report.append(service.clone()); } execution }, stats_summary, stderr, host_guard.command.clone(), host_guard.child.id(), artifacts, relay_config, fabric_home, fabric_invocation, )) } Err(error) => { if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" ) { invalidate_timed_out_local_host(&runtime.runtime_id, &host, &mut host_guard); } Err(error) } } }; let ( adapter_output, resource_snapshot, system_stats, execution, stats_summary, stderr, host_command, host_pid, mut artifacts, relay_config, fabric_home, fabric_invocation, ) = exchange_result?; let agent_result: AgentRunResult = serde_json::from_value(adapter_output).map_err(|error| { lifecycle_error( operation, &runtime.runtime_id, "invalid_agent_run_result", format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) })?; agent_result.validate().map_err(|error| { lifecycle_error( operation, &runtime.runtime_id, "invalid_agent_run_result", format!("adapter returned an invalid AgentRunResult: {error}"), &stderr, ) })?; validate_agent_run_result_extensions(&agent_result, plan.adapter_descriptor.as_ref())?; if !agent_result.artifacts.is_empty() && artifacts.root.is_none() { return Err(lifecycle_error( operation, &runtime.runtime_id, "invalid_agent_run_result", "adapter returned artifacts without a configured runtime artifact root", &stderr, )); } let output = agent_result.output.clone(); let mut events = vec![event_with_metadata( "runtime_start", format!("started runtime {}", runtime.runtime_id), BTreeMap::from([ ( "runtime_id".to_string(), Value::String(runtime.runtime_id.clone()), ), ( "environment_id".to_string(), Value::String(runtime.environment.environment_id.clone()), ), ( "environment_provider".to_string(), Value::String(runtime.environment.provider.clone()), ), ("host_pid".to_string(), Value::from(host_pid)), ]), )]; events.push(event_with_metadata( "invocation_start", format!("invoking persistent local host for {}", harness(plan)), BTreeMap::from([ ( "runtime_id".to_string(), Value::String(runtime.runtime_id.clone()), ), ( "invocation_id".to_string(), Value::String(invocation.invocation_id.clone()), ), ]), )); let (status, error) = agent_result_status(&agent_result); events.push(event_with_metadata( "invocation_end", format!("persistent local host completed with status {status:?}"), BTreeMap::from([ ( "runtime_id".to_string(), Value::String(runtime.runtime_id.clone()), ), ( "invocation_id".to_string(), Value::String(invocation.invocation_id.clone()), ), ]), )); let mut stdout = serde_json::to_string(&output).map_err(FabricError::SerializeJson)?; stdout.push('\n'); write_artifact( &mut artifacts, &fabric_home, "stdout", "log", "stdout.txt", &stdout, "text/plain", )?; if !stderr.is_empty() { write_artifact( &mut artifacts, &fabric_home, "stderr", "log", "stderr.txt", &stderr, "text/plain", )?; } if let Some(sampled) = system_stats .as_ref() .and_then(|stats| stats.sampled.as_ref()) { if let Some(samples) = &sampled.samples { artifacts.artifacts.push(samples.clone()); } } match stats_summary { Some(Ok(reference)) => artifacts.artifacts.push(reference), Some(Err(_)) => events.push(event_with_metadata( "system_stats_unavailable", "system stats summary could not be persisted", BTreeMap::from([( "code".into(), Value::String("stats_summary_write_failed".into()), )]), )), None => {} } collect_workspace_artifacts(&mut artifacts, &fabric_home, runtime, &mut events)?; promote_agent_artifacts_to_manifest(&agent_result.artifacts, &mut artifacts); promote_relay_artifacts_to_manifest(&output, &mut artifacts); let mut metadata = BTreeMap::from([ ( "adapter_runner".to_string(), Value::String("persistent_local_host".to_string()), ), ("host_command".to_string(), Value::String(host_command)), ("host_pid".to_string(), Value::from(host_pid)), ( "fabric_home".to_string(), Value::String(fabric_home.to_string_lossy().into_owned()), ), ( "fabric_invocation".to_string(), Value::String(fabric_invocation.to_string_lossy().into_owned()), ), ( "environment_provider".to_string(), Value::String(runtime.environment.provider.clone()), ), ]); if !agent_result.extensions.is_empty() { metadata.insert( "adapter".to_string(), serde_json::to_value(&agent_result.extensions).map_err(FabricError::SerializeJson)?, ); } Ok(RunResult { agent_name: plan.agent_name.clone(), harness: harness(plan), adapter_kind: adapter_kind(plan), adapter_id: adapter_id(plan), runtime_id: invocation.runtime_id, invocation_id: invocation.invocation_id, request_id: request.request_id, status, output, error, usage: agent_result.usage.as_ref().map(run_usage), execution, resource_snapshot, system_stats, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()) .into_iter() .collect(), events, metadata, }) } fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, host: &mut LocalAdapterHost, ) { { let mut hosts = local_hosts(); if hosts .get(runtime_id) .is_some_and(|host| Arc::ptr_eq(host, expected_host)) { hosts.remove(runtime_id); } } let _ = terminate_local_host(host); let _ = remove_local_host_files(host); } fn agent_result_status(result: &AgentRunResult) -> (RunStatus, Option) { let status = match result.status { AgentRunStatus::Succeeded => RunStatus::Succeeded, AgentRunStatus::Failed => RunStatus::Failed, AgentRunStatus::Cancelled => RunStatus::Cancelled, }; (status, result.error.as_ref().map(error_info)) } fn error_info(error: &AgentRunError) -> ErrorInfo { ErrorInfo { stage: ErrorStage::Invoke, code: error.code.clone(), message: error.message.clone(), retryable: error.retryable, metadata: error.extensions.clone(), } } fn run_usage(usage: &AgentUsage) -> RunUsage { RunUsage { input_tokens: usage.input_tokens, cached_input_tokens: usage.cached_input_tokens, input_tokens_include_cache: usage.input_tokens_include_cache, output_tokens: usage.output_tokens, total_tokens: usage.total_tokens, cost_usd: usage.cost_usd, metadata: usage.extensions.clone(), } } fn promote_agent_artifacts_to_manifest( agent_artifacts: &[AgentArtifact], manifest: &mut ArtifactManifest, ) { for artifact in agent_artifacts { let path = resolve_relay_artifact_path(manifest, &artifact.path); if manifest .artifacts .iter() .any(|existing| existing.path == path) { continue; } let name = unique_artifact_name(manifest, &artifact.name); manifest.artifacts.push(ArtifactRef { name, kind: artifact.kind.clone(), path, media_type: artifact.media_type.clone(), metadata: artifact.extensions.clone(), }); } } fn local_host_stop_event( runtime: &RuntimeHandle, already_stopped: bool, host_crashed: bool, ) -> FabricEvent { event_with_metadata( "runtime_stop", format!("stopped runtime {}", runtime.runtime_id), BTreeMap::from([ ( "runtime_id".to_string(), Value::String(runtime.runtime_id.clone()), ), ("already_stopped".to_string(), Value::Bool(already_stopped)), ("host_crashed".to_string(), Value::Bool(host_crashed)), ]), ) } fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: record.runtime.runtime_id.clone(), })); let result = exchange_lifecycle_message( &mut record.host, &record.runtime.runtime_id, &request, LOCAL_HOST_STOP_TIMEOUT, ); let termination = terminate_local_host(&mut record.host); let diagnostics = local_host_diagnostics(&record.host); let removal = remove_local_host_files(&record.host); let host_crashed = matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" ); if !host_crashed { result?; } termination.map_err(|source| { lifecycle_error( AdapterLifecycleOperation::Stop, &record.runtime.runtime_id, "host_termination_failed", format!("persistent service host could not be terminated: {source}"), diagnostics, ) })?; removal.map_err(|source| { lifecycle_error( AdapterLifecycleOperation::Stop, &record.runtime.runtime_id, "host_cleanup_failed", format!("persistent service host files could not be removed: {source}"), "", ) }) } fn service_release_event(service: &ServiceHandle, already_released: bool) -> FabricEvent { event_with_metadata( "service_release", format!("released service {}", service.service_id), BTreeMap::from([ ( "service_id".to_string(), Value::String(service.service_id.clone()), ), ( "already_released".to_string(), Value::Bool(already_released), ), ( "ownership".to_string(), Value::String( match service.ownership { ServiceOwnership::FabricOwned => "fabric_owned", ServiceOwnership::CallerOwned => "caller_owned", } .to_string(), ), ), ]), ) } // Keep identity-checked snapshots even if interruption fails before the host lock is released. static STOPPING_PROCESSES: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_HOSTS .lock() .unwrap_or_else(|error| error.into_inner()) } fn local_host_processes() -> std::sync::MutexGuard<'static, BTreeMap> { LOCAL_HOST_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) } /// Read retained host exit evidence without taking the invocation lock. /// Release the process registry before inspecting the process. pub(crate) fn local_host_has_exited(runtime_id: &str) -> Option { #[cfg(windows)] { let control = windows_host_controls().get(runtime_id).cloned()?; control.has_exited() } #[cfg(not(windows))] { let process = local_host_processes().get(runtime_id).cloned()?; process.has_exited() } } // Keep owned handles outside the invocation lock and after timeout eviction. // Handles continue to identify the same kernel objects even if PIDs are reused. #[cfg(windows)] struct WindowsHostControl { process: std::os::windows::io::OwnedHandle, job: Option, } #[cfg(windows)] impl WindowsHostControl { fn capture(host: &LocalAdapterHost) -> std::io::Result { use std::os::windows::io::AsHandle; Ok(Self { process: host.child.as_handle().try_clone_to_owned()?, job: host.job.as_ref().map(|job| job.try_clone()).transpose()?, }) } fn has_exited(&self) -> Option { use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; // SAFETY: this owned process handle stays valid throughout the probe. match unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject( self.process.as_raw_handle(), 0, ) } { WAIT_OBJECT_0 => Some(true), WAIT_TIMEOUT => Some(false), _ => None, } } fn cleanup_confirmed(&self) -> Option { use std::os::windows::io::AsRawHandle; use windows_sys::Win32::System::JobObjects::{ JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, JobObjectBasicAccountingInformation, QueryInformationJobObject, }; match self.has_exited() { Some(true) => {} other => return other, } let Some(job) = &self.job else { return None; }; // SAFETY: this is a plain output structure with no invalid bit patterns. let mut accounting: JOBOBJECT_BASIC_ACCOUNTING_INFORMATION = unsafe { std::mem::zeroed() }; // SAFETY: the job handle is owned; the writable buffer matches the class and size. let queried = unsafe { QueryInformationJobObject( job.as_raw_handle(), JobObjectBasicAccountingInformation, (&mut accounting as *mut JOBOBJECT_BASIC_ACCOUNTING_INFORMATION).cast(), std::mem::size_of_val(&accounting) as u32, std::ptr::null_mut(), ) }; (queried != 0).then_some(accounting.ActiveProcesses == 0) } fn interrupt(&self) -> std::io::Result<()> { use std::os::windows::io::AsRawHandle; let job = self .job .as_ref() .ok_or_else(|| std::io::Error::other("owned Windows process tree is unavailable"))?; // SAFETY: this job handle belongs to the runtime, including its descendants. if unsafe { windows_sys::Win32::System::JobObjects::TerminateJobObject(job.as_raw_handle(), 1) } == 0 { return Err(std::io::Error::last_os_error()); } Ok(()) } } #[cfg(windows)] fn windows_host_controls() -> std::sync::MutexGuard<'static, BTreeMap>> { WINDOWS_HOST_CONTROLS .lock() .unwrap_or_else(|error| error.into_inner()) } fn local_services() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_SERVICES .lock() .unwrap_or_else(|error| error.into_inner()) } pub(crate) fn local_host_directory(id: &str) -> PathBuf { std::env::temp_dir() .join("nemo-fabric") .join("hosts") .join(id) } fn spawn_local_host( plan: &RunPlan, runtime: &RuntimeHandle, artifacts: ArtifactManifest, relay_config: Option, environment_env: BTreeMap, service_state_lock: Option, ) -> Result { // A connected host inherits a duplicate of its service's lock, so either // host can finish cleanup without releasing the other host's protection. let state_lock = match service_state_lock { Some(lock) => Some(lock), None if runtime.service_id.is_none() => lock_agent_state(plan, &artifacts)?, None => None, }; let runtime_dir = local_host_directory(&runtime.runtime_id); std::fs::create_dir_all(&runtime_dir).map_err(|source| FabricError::Write { path: runtime_dir.clone(), source, })?; let stderr_path = runtime_dir.join("host.stderr.log"); let stderr = File::create(&stderr_path).map_err(|source| FabricError::Write { path: stderr_path.clone(), source, })?; let (mut command, command_display) = match local_host_command(plan, runtime) { Ok(command) => command, Err(error) => { let _ = std::fs::remove_dir_all(&runtime_dir); return Err(error); } }; command .envs(&environment_env) .env("FABRIC_RUNTIME_ID", &runtime.runtime_id) .env("FABRIC_HOME", &runtime_dir) .envs(relay_env(&relay_config)) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::from(stderr)); // The host inherits the state lock, so the state stays locked until the // host exits too, even when this process exits or crashes first. #[cfg(unix)] if let Some(lock) = &state_lock { use std::os::fd::AsRawFd; use std::os::unix::process::CommandExt; let fd = lock.as_raw_fd(); // SAFETY: the closure runs between fork and exec and only calls // fcntl, which is async-signal-safe. unsafe { command.pre_exec(move || { if libc::fcntl(fd, libc::F_SETFD, 0) == -1 { return Err(std::io::Error::last_os_error()); } Ok(()) }); } } if let Some(root) = artifacts.root.as_ref() { command.env("FABRIC_ARTIFACTS", root); } // Start suspended so the host joins its job before it can start a process. #[cfg(windows)] std::os::windows::process::CommandExt::creation_flags( &mut command, windows_sys::Win32::System::Threading::CREATE_SUSPENDED, ); let mut child = match command.spawn() { Ok(child) => child, Err(source) => { let _ = std::fs::remove_dir_all(&runtime_dir); return Err(FabricError::ProcessRunner { command: command_display, source, }); } }; #[cfg(windows)] let job = host_job(&child); #[cfg(windows)] if let Err(source) = resume_host(child.id()) { let _ = child.kill(); let _ = child.wait(); let _ = std::fs::remove_dir_all(&runtime_dir); return Err(FabricError::ProcessRunner { command: command_display, source, }); } let Some(stdin) = child.stdin.take() else { let _ = child.kill(); let _ = child.wait(); let _ = std::fs::remove_dir_all(&runtime_dir); return Err(lifecycle_error( AdapterLifecycleOperation::Start, &runtime.runtime_id, "host_io", "persistent local adapter host stdin was not available", "", )); }; let Some(stdout) = child.stdout.take() else { let _ = child.kill(); let _ = child.wait(); let _ = std::fs::remove_dir_all(&runtime_dir); return Err(lifecycle_error( AdapterLifecycleOperation::Start, &runtime.runtime_id, "host_io", "persistent local adapter host stdout was not available", "", )); }; let (sender, responses) = mpsc::channel(); if let Err(source) = thread::Builder::new() .name(format!("fabric-host-{}", runtime.runtime_id)) .spawn(move || { let mut stdout = BufReader::new(stdout); loop { let mut line = String::new(); match stdout.read_line(&mut line) { Ok(0) => break, Ok(_) => { while line.ends_with(['\n', '\r']) { line.pop(); } if sender.send(Ok(line)).is_err() { break; } } Err(error) => { let _ = sender.send(Err(error.to_string())); break; } } } }) { let _ = child.kill(); let _ = child.wait(); let _ = std::fs::remove_dir_all(&runtime_dir); return Err(FabricError::ProcessRunner { command: command_display, source, }); } Ok(LocalAdapterHost { service_execution: None, execution: crate::provenance::core_snapshot(&plan.config.provenance), adapter_execution: None, child, pending_stop: Vec::new(), #[cfg(windows)] job, stdin, responses, command: command_display, runtime_dir, stderr_path, stderr_offset: 0, artifacts, relay_config, environment_env, _state_lock: state_lock, }) } fn local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { match adapter_kind(plan) { AdapterKind::Process => process_local_host_command(plan, runtime), AdapterKind::Python => python_local_host_command(plan, runtime), adapter_kind => Err(FabricError::UnsupportedRuntimeAdapter { harness: harness(plan), adapter_kind, }), } } fn process_local_host_command( plan: &RunPlan, runtime: &RuntimeHandle, ) -> Result<(Command, String)> { let settings = parse_process_settings(plan)?; let command_path = resolve_command_path( adapter_setting_root(plan, "command"), Path::new(&settings.command), ); let command_args = process_command_args(plan, &settings); let cwd = settings .cwd .as_ref() .map(|path| resolve_path(&plan.base_dir, path)) .or_else(|| runtime.environment.workspace.clone()) .unwrap_or_else(|| plan.base_dir.clone()); let mut command = Command::new(&command_path); command .args(&command_args) .current_dir(cwd) .envs(&settings.env) .envs(&runtime.environment.env); let display = std::iter::once(command_path.to_string_lossy().into_owned()) .chain(command_args) .collect::>() .join(" "); Ok((command, display)) } fn python_local_host_command(plan: &RunPlan, runtime: &RuntimeHandle) -> Result<(Command, String)> { let settings = parse_python_settings(plan)?; let python = resolve_python_command(&plan.base_dir, &settings).path; let cwd = settings .cwd .as_ref() .map(|path| resolve_path(&plan.base_dir, path)) .or_else(|| runtime.environment.workspace.clone()) .unwrap_or_else(|| plan.base_dir.clone()); let mut command = Command::new(&python); command .arg("-m") .arg(&settings.module) .args(&settings.args) .current_dir(cwd) .envs(&settings.env) .envs(&runtime.environment.env); Ok(( command, format!("{} -m {}", python.to_string_lossy(), settings.module), )) } fn exchange_lifecycle_message( host: &mut LocalAdapterHost, runtime_id: &str, request: &AdapterLifecycleRequest, timeout: Duration, ) -> Result { let operation = request.operation(); if let Some(status) = host.child.try_wait().map_err(|source| { lifecycle_error( operation, runtime_id, "host_io", format!("failed to inspect persistent local adapter host: {source}"), local_host_diagnostics(host), ) })? { return Err(lifecycle_error( operation, runtime_id, "host_crashed", format!( "persistent local adapter host exited before {} ({status})", operation.as_str() ), local_host_diagnostics(host), )); } let mut message = serde_json::to_string(request).map_err(FabricError::SerializeJson)?; message.push('\n'); if let Err(source) = host .stdin .write_all(message.as_bytes()) .and_then(|()| host.stdin.flush()) { let code = match (source.kind(), host.child.try_wait()) { // A closed lifecycle pipe is terminal even when the child exit // status is not observable yet. (ErrorKind::BrokenPipe, _) | (_, Ok(Some(_))) => "host_crashed", _ => "host_io", }; return Err(lifecycle_error( operation, runtime_id, code, format!( "failed to send {} to persistent local adapter host: {source}", operation.as_str() ), local_host_diagnostics(host), )); } let line = match host.responses.recv_timeout(timeout) { Ok(line) => line, Err(RecvTimeoutError::Timeout) => { return Err(lifecycle_error( operation, runtime_id, "host_timeout", format!( "persistent local adapter host did not complete {} within {} ms", operation.as_str(), timeout.as_millis() ), local_host_diagnostics(host), )); } Err(RecvTimeoutError::Disconnected) => { return Err(lifecycle_error( operation, runtime_id, "host_crashed", format!( "persistent local adapter host exited while processing {}", operation.as_str() ), local_host_diagnostics(host), )); } } .map_err(|message| { lifecycle_error( operation, runtime_id, "host_io", message, local_host_diagnostics(host), ) })?; let response: AdapterLifecycleResponse = serde_json::from_str(&line).map_err(|source| { lifecycle_error( operation, runtime_id, "protocol_error", format!("invalid lifecycle response: {source}"), local_host_diagnostics(host), ) })?; if response.operation != operation { return Err(lifecycle_error( operation, runtime_id, "protocol_error", format!( "expected `{}` response but host returned `{}`", operation.as_str(), response.operation.as_str() ), local_host_diagnostics(host), )); } if host.execution.is_some() && operation != AdapterLifecycleOperation::Stop && (response.execution.is_some() || host.adapter_execution.is_none()) { let mut report = crate::ExecutionEvidence::receive(response.execution, "adapter"); if !report .software_environments .iter() .any(|env| env.participants.iter().any(|p| p == "adapter")) { report.append(crate::ExecutionEvidence::gap( "adapter", "expected_adapter_report_missing", )); } host.adapter_execution = Some(report); } match response.outcome { AdapterLifecycleOutcome::Succeeded { output } => Ok(output), AdapterLifecycleOutcome::Failed { error } => { if error.stage != operation.error_stage() { return Err(lifecycle_error( operation, runtime_id, "protocol_error", format!( "{} failure reported the wrong lifecycle stage `{:?}`", operation.as_str(), error.stage ), local_host_diagnostics(host), )); } let mut diagnostics = local_host_diagnostics(host); if !error.metadata.is_empty() && let Ok(metadata) = serde_json::to_string(&error.metadata) { if !diagnostics.is_empty() { diagnostics.push('\n'); } diagnostics.push_str("adapter metadata: "); diagnostics.push_str(&metadata); } Err(lifecycle_error( operation, runtime_id, error.code, error.message, diagnostics, )) } } } fn lifecycle_error( operation: AdapterLifecycleOperation, runtime_id: &str, code: impl Into, message: impl Into, diagnostics: impl Into, ) -> FabricError { FabricError::AdapterLifecycleOperation { operation: operation.as_str(), runtime_id: runtime_id.to_string(), code: code.into(), message: message.into(), diagnostics: diagnostics.into(), } } fn local_host_diagnostics(host: &LocalAdapterHost) -> String { let Ok(bytes) = std::fs::read(&host.stderr_path) else { return String::new(); }; let start = bytes.len().saturating_sub(LOCAL_HOST_DIAGNOSTIC_LIMIT); String::from_utf8_lossy(&bytes[start..]).trim().to_string() } fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { let Ok(bytes) = std::fs::read(&host.stderr_path) else { return String::new(); }; let start = host.stderr_offset.min(bytes.len()); host.stderr_offset = bytes.len(); String::from_utf8_lossy(&bytes[start..]).into_owned() } fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { if host.child.try_wait()?.is_some() { return Ok(()); } // Snapshot before the grace wait: once the host exits, its children are // re-parented and can no longer be found from its PID. #[cfg(unix)] let mut descendants = host_descendants(host.child.id()); let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; loop { if host.child.try_wait()?.is_some() { #[cfg(unix)] kill_pids(&descendants); return Ok(()); } if Instant::now() >= deadline { break; } thread::sleep(Duration::from_millis(10)); } #[cfg(unix)] { // Also reach processes the host started during the grace wait. descendants.extend(host_descendants(host.child.id())); kill_pids(&descendants); } #[cfg(windows)] if let Some(job) = &host.job // SAFETY: the host owns this job handle until it is dropped. && unsafe { windows_sys::Win32::System::JobObjects::TerminateJobObject( std::os::windows::io::AsRawHandle::as_raw_handle(job), 1, ) } != 0 { host.child.wait()?; return Ok(()); } if let Err(source) = host.child.kill() && host.child.try_wait()?.is_none() { return Err(source); } host.child.wait()?; Ok(()) } /// Lists every live descendant of a host, including children that started /// their own session. Processes that already detached from the host's process /// tree cannot be found this way, so cleanup stays best-effort. #[cfg(unix)] fn host_descendants(host_pid: u32) -> Vec { let Ok(host_pid) = libc::pid_t::try_from(host_pid) else { return Vec::new(); }; // Snapshot the whole tree first: killed processes' children are re-parented. let mut descendants = Vec::new(); let mut parents = vec![host_pid]; while let Some(parent) = parents.pop() { let children = child_pids(parent); parents.extend(&children); descendants.extend(children); } descendants } #[cfg(unix)] fn kill_pids(pids: &[libc::pid_t]) { for &pid in pids { // SAFETY: kill has no memory-safety preconditions. PIDs are allocated // sequentially, so reuse within the exit grace is not a practical risk. unsafe { libc::kill(pid, libc::SIGKILL) }; } } #[cfg(target_os = "linux")] fn child_pids(parent: libc::pid_t) -> Vec { let Ok(entries) = std::fs::read_dir("/proc") else { return Vec::new(); }; entries .flatten() .filter_map(|entry| { let pid = entry.file_name().to_str()?.parse().ok()?; let stat = std::fs::read_to_string(entry.path().join("stat")).ok()?; // The command name may contain spaces or parentheses; the parent PID // is the second field after it. let ppid: libc::pid_t = stat .rsplit_once(')')? .1 .split_whitespace() .nth(1)? .parse() .ok()?; (ppid == parent).then_some(pid) }) .collect() } #[cfg(target_os = "macos")] fn child_pids(parent: libc::pid_t) -> Vec { let mut children = vec![0; 64]; loop { let Ok(bytes) = i32::try_from(std::mem::size_of_val(children.as_slice())) else { return Vec::new(); }; // SAFETY: the buffer is writable and sized in bytes as libproc requires. let count = unsafe { libc::proc_listchildpids(parent, children.as_mut_ptr().cast(), bytes) }; let Ok(count) = usize::try_from(count) else { return Vec::new(); }; if count < children.len() { children.truncate(count); return children; } children.resize(children.len() * 2, 0); } } #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] fn child_pids(_parent: libc::pid_t) -> Vec { Vec::new() } /// Puts the suspended host in a job object so a forced kill reaches every /// process it starts, even through launchers or other intermediates that /// already exited. Like Cargo, fall back to killing only the host if the job /// is unavailable. #[cfg(windows)] fn host_job(child: &Child) -> Option { use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; use windows_sys::Win32::System::JobObjects::{AssignProcessToJobObject, CreateJobObjectW}; // SAFETY: null security attributes and name are allowed; null means failure. let job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; if job.is_null() { return None; } // SAFETY: CreateJobObjectW returned a new handle that nothing else owns. let job = unsafe { OwnedHandle::from_raw_handle(job) }; // SAFETY: both handles stay valid for the duration of the call. let assigned = unsafe { AssignProcessToJobObject(job.as_raw_handle(), child.as_raw_handle()) }; (assigned != 0).then_some(job) } /// Resumes the main thread of a host spawned with `CREATE_SUSPENDED`. #[cfg(windows)] fn resume_host(pid: u32) -> std::io::Result<()> { use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; use windows_sys::Win32::System::Diagnostics::ToolHelp::{ CreateToolhelp32Snapshot, TH32CS_SNAPTHREAD, THREADENTRY32, Thread32First, Thread32Next, }; use windows_sys::Win32::System::Threading::{OpenThread, ResumeThread, THREAD_SUSPEND_RESUME}; // SAFETY: a thread snapshot takes no pointers; failure is reported below. let snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0) }; if snapshot == INVALID_HANDLE_VALUE { return Err(std::io::Error::last_os_error()); } // SAFETY: the snapshot handle is new and owned only here. let snapshot = unsafe { OwnedHandle::from_raw_handle(snapshot) }; // SAFETY: THREADENTRY32 is plain data; zero is a valid initial value. let mut entry: THREADENTRY32 = unsafe { std::mem::zeroed() }; entry.dwSize = std::mem::size_of::() as u32; // SAFETY: `entry` is a writable THREADENTRY32 with `dwSize` set. let mut found = unsafe { Thread32First(snapshot.as_raw_handle(), &mut entry) } != 0; let mut resumed = false; while found { if entry.th32OwnerProcessID == pid { // SAFETY: OpenThread takes no pointers; null means failure. let thread = unsafe { OpenThread(THREAD_SUSPEND_RESUME, 0, entry.th32ThreadID) }; if thread.is_null() { return Err(std::io::Error::last_os_error()); } // SAFETY: OpenThread returned a new handle owned only here. let thread = unsafe { OwnedHandle::from_raw_handle(thread) }; // SAFETY: the handle has THREAD_SUSPEND_RESUME access. if unsafe { ResumeThread(thread.as_raw_handle()) } == u32::MAX { return Err(std::io::Error::last_os_error()); } resumed = true; } // SAFETY: same buffer as above. found = unsafe { Thread32Next(snapshot.as_raw_handle(), &mut entry) } != 0; } if resumed { Ok(()) } else { // A host left suspended would hang until the start timeout. Err(std::io::Error::other("suspended host thread was not found")) } } fn remove_local_host_files(host: &LocalAdapterHost) -> std::io::Result<()> { match std::fs::remove_dir_all(&host.runtime_dir) { Ok(()) => Ok(()), Err(source) if source.kind() == ErrorKind::NotFound => Ok(()), Err(source) => Err(source), } } fn adapter_id(plan: &RunPlan) -> Option { plan.adapter_descriptor .as_ref() .map(|adapter| adapter.descriptor.adapter_id.clone()) .or_else(|| { plan.config .harness .as_ref() .map(|harness| harness.adapter_id.clone()) }) } fn harness(plan: &RunPlan) -> String { adapter_id(plan).unwrap_or_else(|| "unknown".to_string()) } fn adapter_kind(plan: &RunPlan) -> AdapterKind { plan.adapter_descriptor .as_ref() .map(|adapter| adapter.descriptor.adapter_kind) .unwrap_or(AdapterKind::Process) } fn adapter_kind_name(adapter_kind: AdapterKind) -> String { match adapter_kind { AdapterKind::Process => "process", AdapterKind::Http => "http", AdapterKind::Python => "python", AdapterKind::NativePlugin => "native_plugin", } .to_string() } fn optional_runtime_value(value: Option<&str>) -> String { value.unwrap_or("").to_string() } fn parse_python_settings(plan: &RunPlan) -> Result { let value = Value::Object(merged_adapter_settings(plan)); serde_json::from_value(value).map_err(|source| FabricError::InvalidPythonSettings { path: plan.base_dir.clone(), source, }) } fn parse_process_settings(plan: &RunPlan) -> Result { let value = Value::Object(merged_adapter_settings(plan)); serde_json::from_value(value).map_err(|source| FabricError::InvalidProcessSettings { path: plan.base_dir.clone(), source, }) } fn merged_adapter_settings(plan: &RunPlan) -> Map { let mut settings = plan .adapter_descriptor .as_ref() .map(|adapter| adapter.descriptor.runner.clone()) .unwrap_or_default(); if let Some(harness) = &plan.config.harness { settings.extend(harness.settings.clone()); } settings } fn adapter_setting_root<'a>(plan: &'a RunPlan, key: &str) -> &'a Path { if plan .config .harness .as_ref() .is_some_and(|harness| harness.settings.contains_key(key)) { return &plan.base_dir; } plan.adapter_descriptor .as_ref() .map(|adapter| adapter.primary().root.as_path()) .unwrap_or(&plan.base_dir) } fn adapter_lifecycle_start( plan: &RunPlan, runtime: &RuntimeHandle, invocation: &InvocationHandle, artifacts: &ArtifactManifest, relay_config: Option<&RelayRuntimeConfig>, service: Option, ) -> Result { let config = adapter_lifecycle_config(plan); Ok(AdapterLifecycleStart { provenance: crate::ProvenanceConfig { enabled: plan.config.provenance.enabled, reports: None, }, agent_name: plan.agent_name.clone(), base_dir: absolute_path(plan.base_dir.clone())?, config, runtime_context: adapter_runtime_context( plan, runtime, invocation, artifacts, relay_config, )?, capability_plan: plan.capability_plan.clone(), telemetry_plan: plan.telemetry_plan.clone(), service, }) } fn adapter_lifecycle_config(plan: &RunPlan) -> AgentConfig { plan.agent_config.clone() } fn adapter_invocation( plan: &RunPlan, runtime: &RuntimeHandle, invocation: &InvocationHandle, request: &RunRequest, artifacts: &ArtifactManifest, relay_config: Option<&RelayRuntimeConfig>, ) -> Result { let request = project_agent_run_request(request)?; validate_agent_run_request_extensions(&request, plan.adapter_descriptor.as_ref())?; Ok(AdapterInvocation { runtime_context: adapter_runtime_context( plan, runtime, invocation, artifacts, relay_config, )?, request, }) } fn project_agent_run_request(request: &RunRequest) -> Result { let extensions = match &request.overrides { Some(Value::Object(extensions)) => extensions .iter() .map(|(key, value)| (key.clone(), value.clone())) .collect(), Some(_) => { return Err(FabricError::InvalidConfig { field: "request.overrides".to_string(), reason: "must be an object".to_string(), }); } None => BTreeMap::new(), }; Ok(AgentRunRequest { input: request.input.clone(), relay_session_root: request.relay_session_root.clone(), context: request.context.clone(), extensions, }) } fn redact_adapter_invocation(invocation: &mut AdapterInvocation) { redact_runtime_context_environment(&mut invocation.runtime_context); } fn redact_openai_stream_invocation(invocation: &mut OpenAiStreamInvocation) { redact_runtime_context_environment(&mut invocation.runtime_context); invocation.stream.token = "[REDACTED]".to_string(); } fn redact_runtime_context_environment(context: &mut RuntimeContext) { for value in context.environment.env.values_mut() { *value = "[REDACTED]".to_string(); } } fn openai_stream_sink( runtime: &RuntimeHandle, invocation: &InvocationHandle, transport: OpenAiStreamTransport, ) -> OpenAiStreamSink { OpenAiStreamSink { protocol_version: OpenAiStreamProtocolVersion::V1Alpha1, profile: OpenAiStreamProfile::ChatCompletionsChunkV1, host: OpenAiStreamHost::Ipv4Loopback, port: transport.port, token: transport.token, runtime_id: runtime.runtime_id.clone(), invocation_id: invocation.invocation_id.clone(), request_id: invocation.request_id.clone(), } } fn validate_named_environment( plan: &RunPlan, startup: &BTreeMap, current: &BTreeMap, ) -> Result<()> { for name in plan .environment_plan .iter() .flat_map(|environment| &environment.env_names) { if startup.get(name) != current.get(name) { return Err(FabricError::InvalidConfig { field: "environment.env_names".to_string(), reason: format!( "environment variable `{name}` changed after adapter start; restart the runtime and any associated service to use its new value" ), }); } } Ok(()) } fn adapter_runtime_context( plan: &RunPlan, runtime: &RuntimeHandle, invocation: &InvocationHandle, artifacts: &ArtifactManifest, relay_config: Option<&RelayRuntimeConfig>, ) -> Result { // Named values stay out of the plan and runtime handle. The adapter-facing // context is redacted before it is persisted. let mut environment = runtime.environment.clone(); resolve_named_environment(plan, &mut environment.env)?; Ok(RuntimeContext { runtime_id: runtime.runtime_id.clone(), invocation_id: invocation.invocation_id.clone(), request_id: invocation.request_id.clone(), environment, artifacts: artifacts.clone(), telemetry: runtime_telemetry_context(plan, relay_config), deadline_millis: None, state_dir: agent_state_dir(plan, artifacts)?, }) } /// The folder that keeps an agent's native state across its runtimes when /// `runtime.keep_state` is set: `/.fabric/state//`. /// The adapter ID keeps another harness from reading the state if the agent's /// adapter changes. fn agent_state_dir(plan: &RunPlan, artifacts: &ArtifactManifest) -> Result> { if !plan.config.runtime.keep_state { return Ok(None); } let Some(root) = artifacts.root.as_ref() else { return Err(FabricError::InvalidConfig { field: "runtime.keep_state".to_string(), reason: "requires an artifact directory, from runtime.artifacts or the environment" .to_string(), }); }; // A deserialized plan can carry any agent name, so check it again here. if !is_one_folder_name(&plan.agent_name) { return Err(FabricError::InvalidConfig { field: "agent_name".to_string(), reason: "must be one folder name when runtime.keep_state is set".to_string(), }); } // A local descriptor can declare any adapter ID, so check it the same way. let adapter = adapter_id(plan).unwrap_or_else(|| harness(plan)); if !is_one_folder_name(&adapter) { return Err(FabricError::InvalidConfig { field: "adapter_id".to_string(), reason: "must be one folder name when runtime.keep_state is set".to_string(), }); } absolute_path( root.join(".fabric") .join("state") .join(adapter) .join(&plan.agent_name), ) .map(Some) } /// Takes the agent's state folder for one runtime. The lock lasts while the /// returned file is open here or, on Unix, in an adapter host that inherits /// it. The operating system releases the lock after all descriptors close. fn lock_agent_state(plan: &RunPlan, artifacts: &ArtifactManifest) -> Result> { let Some(state_dir) = agent_state_dir(plan, artifacts)? else { return Ok(None); }; std::fs::create_dir_all(&state_dir).map_err(|source| FabricError::Write { path: state_dir.clone(), source, })?; // Kept state can hold conversations and credentials: only its owner may // read it, including a folder that an earlier runtime created. #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&state_dir, std::fs::Permissions::from_mode(0o700)).map_err( |source| FabricError::Write { path: state_dir.clone(), source, }, )?; } // The lock sits beside the folder, which the adapter owns and may replace. let mut lock_name = std::ffi::OsString::from("."); lock_name.push(state_dir.file_name().unwrap_or_default()); lock_name.push(".lock"); let lock_path = state_dir.with_file_name(lock_name); let file = File::options() .create(true) .write(true) .truncate(false) .open(&lock_path) .map_err(|source| FabricError::Write { path: lock_path.clone(), source, })?; // A process that forks while another runtime holds the lock, such as one // starting another host, keeps the lock until it execs, so wait briefly // before deciding that the state is in use. let deadline = Instant::now() + STATE_LOCK_WAIT; loop { match file.try_lock() { Ok(()) => return Ok(Some(file)), Err(std::fs::TryLockError::WouldBlock) if Instant::now() < deadline => { thread::sleep(Duration::from_millis(10)); } Err(std::fs::TryLockError::WouldBlock) => { return Err(FabricError::StateInUse { path: state_dir }); } Err(std::fs::TryLockError::Error(source)) => { return Err(FabricError::Write { path: lock_path, source, }); } } } } const STATE_LOCK_WAIT: Duration = Duration::from_millis(250); fn resolve_named_environment(plan: &RunPlan, env: &mut BTreeMap) -> Result<()> { for name in plan .environment_plan .iter() .flat_map(|environment| &environment.env_names) { let value = std::env::var(name).map_err(|error| FabricError::InvalidConfig { field: "environment.env_names".to_string(), reason: match error { std::env::VarError::NotPresent => { format!("environment variable `{name}` is not set") } std::env::VarError::NotUnicode(_) => { format!("environment variable `{name}` is not valid Unicode") } }, })?; env.insert(name.clone(), value); } Ok(()) } /// Converts Fabric's invocation timeout to the adapter's wall-clock deadline, /// keeping a reserve so the adapter can return before the host timeout fires. /// Timeouts under twice the reserve keep only half in reserve; adapter cleanup /// that outlasts it falls back to the host timeout, which kills the host tree. fn adapter_deadline_millis(invoke_timeout: Duration) -> Option { let budget = invoke_timeout .saturating_sub(LOCAL_HOST_ADAPTER_RESERVE) .max(invoke_timeout / 2); let deadline = SystemTime::now().checked_add(budget)?; let millis = deadline.duration_since(UNIX_EPOCH).ok()?.as_millis(); Some(u64::try_from(millis).unwrap_or(u64::MAX)) } fn runtime_telemetry_context( plan: &RunPlan, relay_config: Option<&RelayRuntimeConfig>, ) -> Option { let telemetry = plan.telemetry_plan.as_ref()?; let mut metadata = BTreeMap::new(); metadata.insert( "telemetry_providers".to_string(), Value::Array( telemetry .providers .iter() .map(|provider| Value::String(provider.as_str().to_string())) .collect(), ), ); if let Some(project) = &telemetry.relay_project { metadata.insert("relay_project".to_string(), Value::String(project.clone())); } if let Some(output_dir) = &telemetry.relay_output_dir { metadata.insert( "relay_output_dir".to_string(), Value::String(output_dir.to_string_lossy().into_owned()), ); } if let Some(native_config) = &telemetry.native_config { metadata.insert("native_config".to_string(), native_config.clone()); } if !telemetry.adapter_outputs.is_empty() { metadata.insert( "adapter_outputs".to_string(), Value::Array( telemetry .adapter_outputs .iter() .map(|output| Value::String(output.clone())) .collect(), ), ); } Some(RuntimeTelemetryContext { relay_enabled: telemetry.relay_enabled, config_path: relay_config.map(|relay| relay.path.clone()), env: relay_config .map(|relay| relay.env.clone()) .unwrap_or_default(), metadata, }) } pub(crate) fn resolve_path(root: &Path, path: &Path) -> PathBuf { if path.is_absolute() { return path.to_path_buf(); } root.join(path) } fn resolve_command_path(root: &Path, path: &Path) -> PathBuf { if path.is_absolute() || path.components().count() > 1 { return resolve_path(root, path); } path.to_path_buf() } fn preflight_python_adapter(plan: &RunPlan) -> Result<()> { let settings = parse_python_settings(plan)?; let command = resolve_python_command(&plan.base_dir, &settings); validate_python_command(&command) } fn validate_python_command(command: &PythonCommand) -> Result<()> { let path = &command.path; // A single-component relative path (e.g. `python3`) is a bare command name // resolved off PATH; anything else is a concrete file path we can stat. let is_bare_command = !path.is_absolute() && path.components().count() == 1; if is_bare_command { if find_on_path(path).is_none() { return Err(FabricError::PythonInterpreterUnavailable { path: path.clone(), origin: command.source.describe(), reason: "not found on PATH".to_string(), }); } } else if !path.is_file() { let reason = if path.exists() { "not a regular file" } else { "no such file" }; return Err(FabricError::PythonInterpreterUnavailable { path: path.clone(), origin: command.source.describe(), reason: reason.to_string(), }); } Ok(()) } /// Locate a bare command name on `PATH`, returning the first matching file. fn find_on_path(name: &Path) -> Option { let paths = std::env::var_os("PATH")?; std::env::split_paths(&paths) .map(|dir| dir.join(name)) .find(|candidate| candidate.is_file()) } fn resolve_python_command(root: &Path, settings: &PythonAdapterSettings) -> PythonCommand { resolve_python_command_with_env( root, settings, |name| std::env::var_os(name), |path| path.is_file(), std::env::current_exe().ok(), ) } fn resolve_python_command_with_env( root: &Path, settings: &PythonAdapterSettings, env: impl Fn(&str) -> Option, exists: impl Fn(&Path) -> bool, current_exe: Option, ) -> PythonCommand { if let Some(path) = settings.python.as_ref() { return PythonCommand { path: resolve_command_path(root, path), source: PythonSource::Setting, }; } if let Some(env_name) = settings.python_env.as_ref() { // A set-but-empty variable is treated as unset so it falls through to // the shared fallback chain rather than yielding an empty path. if let Some(path) = env(env_name) && !path.as_os_str().is_empty() { return PythonCommand { path: resolve_command_path(root, Path::new(&path)), source: PythonSource::SettingEnv(env_name.clone()), }; } return fallback_interpreter(&env, &exists, current_exe.as_deref()); } if let Some(value) = env(ADAPTER_PYTHON_ENV) && !value.as_os_str().is_empty() { return PythonCommand { path: resolve_command_path(root, Path::new(&value)), source: PythonSource::AdapterPythonEnv, }; } fallback_interpreter(&env, &exists, current_exe.as_deref()) } /// With no interpreter explicitly configured, prefer (in order) the active /// virtualenv, an interpreter next to the running NeMo Fabric executable, and only /// then a bare `python3` off PATH. A preinstalled adapter is installed in the /// caller's environment, so launching it with an unrelated `python3` off PATH /// otherwise dies mid-run with an opaque ModuleNotFoundError (FABRIC-86). fn fallback_interpreter( env: &impl Fn(&str) -> Option, exists: &impl Fn(&Path) -> bool, current_exe: Option<&Path>, ) -> PythonCommand { if let Some(virtual_env) = env(VIRTUAL_ENV_ENV) && !virtual_env.as_os_str().is_empty() { // Inside an active virtualenv, always target its interpreter. If it is // missing, preflight surfaces a clear PythonInterpreterUnavailable // rather than silently falling back to an unrelated `python3`. return PythonCommand { path: Path::new(&virtual_env).join(VENV_BIN_DIR).join(VENV_PYTHON), source: PythonSource::Virtualenv, }; } if let Some(exe) = current_exe && let Some(dir) = exe.parent() { let candidate = dir.join(VENV_PYTHON); if exists(&candidate) { return PythonCommand { path: candidate, source: PythonSource::HostInterpreter, }; } } PythonCommand { path: PathBuf::from(DEFAULT_PYTHON), source: PythonSource::DefaultPython3, } } pub(crate) fn absolute_path(path: PathBuf) -> Result { if path.is_absolute() { return Ok(path); } let cwd = std::env::current_dir().map_err(|source| FabricError::ProcessRunner { command: "current_dir".to_string(), source, })?; Ok(cwd.join(path)) } #[derive(Debug, Default, Deserialize)] struct RelayArtifactOutput { #[serde(default)] relay_artifacts: Vec, } #[derive(Debug, Deserialize)] struct RelayArtifactCandidate { kind: String, path: PathBuf, format: Option, } fn promote_relay_artifacts_to_manifest(output: &Value, manifest: &mut ArtifactManifest) { let relay_output: RelayArtifactOutput = serde_json::from_value(output.clone()).unwrap_or_default(); for artifact in relay_output.relay_artifacts { let Ok(artifact) = serde_json::from_value::(artifact) else { continue; }; let kind = artifact.kind.as_str(); if !matches!(kind, "atof" | "atif" | "otlp") { continue; } if artifact.path.as_os_str().is_empty() { continue; } let path = resolve_relay_artifact_path(manifest, &artifact.path); if !path.exists() || manifest .artifacts .iter() .any(|artifact| artifact.path == path) { continue; } let name = unique_artifact_name(manifest, &format!("relay_{kind}")); manifest.artifacts.push(ArtifactRef { name, kind: kind.to_string(), path, media_type: relay_artifact_media_type(kind, artifact.format.as_deref()) .map(str::to_string), metadata: BTreeMap::new(), }); } } fn resolve_relay_artifact_path(manifest: &ArtifactManifest, path: &Path) -> PathBuf { if path.is_absolute() { return path.to_path_buf(); } manifest .root .as_ref() .map(|root| root.join(path)) .unwrap_or_else(|| path.to_path_buf()) } fn relay_artifact_media_type(kind: &str, format: Option<&str>) -> Option<&'static str> { match kind { "atof" => Some("application/x-ndjson"), "otlp" if format == Some("proto") => Some("application/x-protobuf"), "otlp" if matches!(format, None | Some("json_lines")) => Some("application/x-ndjson"), "atif" => Some("application/json"), _ => None, } } fn unique_artifact_name(manifest: &ArtifactManifest, base: &str) -> String { if !manifest .artifacts .iter() .any(|artifact| artifact.name == base) { return base.to_string(); } let mut index = 2; loop { let candidate = format!("{base}_{index}"); if !manifest .artifacts .iter() .any(|artifact| artifact.name == candidate) { return candidate; } index += 1; } } fn process_command_args(plan: &RunPlan, settings: &ProcessAdapterSettings) -> Vec { let mut args = Vec::new(); if let Some(script) = settings.script.as_ref() { let root = adapter_setting_root(plan, "script"); args.push(resolve_path(root, script).to_string_lossy().into_owned()); } args.extend(settings.args.clone()); args } fn artifact_manifest(plan: &RunPlan) -> Result { let root = plan .config .runtime .artifacts .as_ref() .map(|path| resolve_path(&plan.base_dir, path)) .or_else(|| { plan.environment_plan .as_ref() .and_then(|environment| environment.artifacts.clone()) }); if let Some(root) = &root { std::fs::create_dir_all(root).map_err(|source| FabricError::Write { path: root.clone(), source, })?; } Ok(ArtifactManifest { root, artifacts: Vec::new(), }) } fn prepare_fabric_home( manifest: &ArtifactManifest, runtime: &RuntimeHandle, invocation: &InvocationHandle, ) -> Result { let root = manifest .root .clone() .unwrap_or_else(|| std::env::temp_dir().join("nemo-fabric")); let fabric_home = root .join(".fabric") .join(&runtime.runtime_id) .join(&invocation.invocation_id); std::fs::create_dir_all(&fabric_home).map_err(|source| FabricError::Write { path: fabric_home.clone(), source, })?; Ok(fabric_home) } fn write_fabric_invocation(fabric_home: &Path, payload: &str) -> Result { let path = fabric_home.join("adapter-invocation.json"); std::fs::write(&path, payload).map_err(|source| FabricError::Write { path: path.clone(), source, })?; Ok(path) } fn write_artifact( manifest: &mut ArtifactManifest, directory: &Path, name: &str, kind: &str, filename: &str, contents: &str, media_type: &str, ) -> Result<()> { if manifest.root.is_none() { return Ok(()); } let path = directory.join(filename); std::fs::write(&path, contents).map_err(|source| FabricError::Write { path: path.clone(), source, })?; manifest.artifacts.push(ArtifactRef { name: name.to_string(), kind: kind.to_string(), path, media_type: Some(media_type.to_string()), metadata: BTreeMap::new(), }); Ok(()) } fn collect_workspace_artifacts( manifest: &mut ArtifactManifest, artifact_directory: &Path, runtime: &RuntimeHandle, events: &mut Vec, ) -> Result<()> { let Some(workspace) = runtime.environment.workspace.as_ref() else { return Ok(()); }; if !workspace.join(".git").exists() { return Ok(()); } let Ok(status) = Command::new("git") .arg("-C") .arg(workspace) .arg("status") .arg("--short") .output() else { return Ok(()); }; if !status.status.success() { return Ok(()); } let status_text = String::from_utf8_lossy(&status.stdout).into_owned(); if status_text.trim().is_empty() { return Ok(()); } // Kept agent state can hold conversations and credentials; leave it out // when the artifact directory sits inside the workspace. let kept_state: Vec = manifest .root .iter() .map(|root| root.join(".fabric").join("state")) .collect(); let Some(mut patch) = tracked_workspace_patch(workspace, &kept_state) else { return Ok(()); }; patch.push_str(&untracked_workspace_patch(workspace, &kept_state)?); if patch.trim().is_empty() { return Ok(()); } write_artifact( manifest, artifact_directory, "workspace_patch", "patch", "workspace.patch", &patch, "text/x-diff", )?; write_artifact( manifest, artifact_directory, "workspace_status", "log", "workspace-status.txt", &status_text, "text/plain", )?; events.push(event_with_metadata( "artifact_collect", "collected workspace patch artifact", BTreeMap::from([ ( "runtime_id".to_string(), Value::String(runtime.runtime_id.clone()), ), ( "workspace".to_string(), Value::String(workspace.to_string_lossy().into_owned()), ), ]), )); Ok(()) } fn tracked_workspace_patch(workspace: &Path, excluded: &[PathBuf]) -> Option { let diff = Command::new("git") .arg("-C") .arg(workspace) .arg("diff") .arg("--binary") .arg("--") .arg(".") .args(excluded_pathspecs(workspace, excluded)) .output() .ok()?; if !diff.status.success() { return None; } Some(String::from_utf8_lossy(&diff.stdout).into_owned()) } /// Git pathspecs that leave out each excluded path inside the workspace. fn excluded_pathspecs(workspace: &Path, excluded: &[PathBuf]) -> Vec { let workspace_root = workspace .canonicalize() .unwrap_or_else(|_| workspace.to_path_buf()); excluded .iter() .filter_map(|path| { let path = path.canonicalize().unwrap_or_else(|_| path.clone()); let relative = path.strip_prefix(&workspace_root).ok()?; // Match the path literally: a folder name such as "artifacts[1]" // would otherwise be a pattern that also leaves out other paths. let mut pathspec = std::ffi::OsString::from(":(exclude,literal)"); pathspec.push(relative); Some(pathspec) }) .collect() } fn untracked_workspace_patch(workspace: &Path, excluded: &[PathBuf]) -> Result { let mut command = Command::new("git"); command .arg("-C") .arg(workspace) .arg("ls-files") .arg("--others") .arg("--exclude-standard") .arg("-z") .arg("--") .arg(".") .args(excluded_pathspecs(workspace, excluded)); let Ok(untracked) = command.output() else { return Ok(String::new()); }; if !untracked.status.success() || untracked.stdout.is_empty() { return Ok(String::new()); } let mut patch = String::new(); for raw_path in untracked.stdout.split(|byte| *byte == 0) { if raw_path.is_empty() { continue; } let relative_path = String::from_utf8_lossy(raw_path); let Ok(diff) = Command::new("git") .arg("-C") .arg(workspace) .arg("diff") .arg("--binary") .arg("--no-index") .arg("--") .arg("/dev/null") .arg(relative_path.as_ref()) .output() else { continue; }; if !diff.stdout.is_empty() { if !patch.is_empty() && !patch.ends_with('\n') { patch.push('\n'); } patch.push_str(&String::from_utf8_lossy(&diff.stdout)); } } Ok(patch) } fn prepare_relay_runtime_config( plan: &RunPlan, runtime: &RuntimeHandle, artifact_directory: &Path, artifacts: &mut ArtifactManifest, ) -> Result> { let Some(telemetry) = plan.telemetry_plan.as_ref() else { return Ok(None); }; if !telemetry.relay_enabled { return Ok(None); } if artifacts.root.is_none() { return Ok(None); } let fabric = serde_json::json!({ "agent_name": plan.agent_name.clone(), "harness": harness(plan), "adapter_id": adapter_id(plan), "runtime_id": runtime.runtime_id.clone(), "adapter_outputs": telemetry.adapter_outputs.clone(), }); let relay_config = serde_json::json!({ "schema_version": "fabric.relay/v1alpha1", "relay": { "enabled": true, "project": telemetry.relay_project.clone(), "output_dir": telemetry .relay_output_dir .as_ref() .map(|path| path.to_string_lossy().into_owned()), "config": telemetry .relay_config .clone() .unwrap_or_else(|| Value::Object(Default::default())), }, "fabric": fabric, }); let contents = serde_json::to_string_pretty(&relay_config).map_err(FabricError::SerializeJson)?; write_artifact( artifacts, artifact_directory, "relay_config", "telemetry_config", "relay-config.json", &contents, "application/json", )?; let path = absolute_path(artifact_directory.join("relay-config.json"))?; Ok(Some(RelayRuntimeConfig { path: path.clone(), env: BTreeMap::from([ ("FABRIC_RELAY_ENABLED".to_string(), "true".to_string()), ( "FABRIC_RELAY_CONFIG_PATH".to_string(), path.to_string_lossy().into_owned(), ), ]), })) } fn relay_env(relay_config: &Option) -> BTreeMap { relay_config .as_ref() .map(|relay| relay.env.clone()) .unwrap_or_default() } fn telemetry_ref( plan: &RunPlan, relay_runtime: Option<&RelayRuntimeConfig>, ) -> Option { let telemetry = plan.telemetry_plan.as_ref()?; let mut metadata = BTreeMap::new(); metadata.insert( "telemetry_providers".to_string(), Value::Array( telemetry .providers .iter() .map(|provider| Value::String(provider.as_str().to_string())) .collect(), ), ); if let Some(project) = &telemetry.relay_project { metadata.insert("relay_project".to_string(), Value::String(project.clone())); } if let Some(output_dir) = &telemetry.relay_output_dir { metadata.insert( "relay_output_dir".to_string(), Value::String(output_dir.to_string_lossy().into_owned()), ); } if let Some(config) = &telemetry.relay_config { metadata.insert("relay_config".to_string(), config.clone()); } if !telemetry.adapter_outputs.is_empty() { metadata.insert( "adapter_outputs".to_string(), Value::Array( telemetry .adapter_outputs .iter() .map(|output| Value::String(output.clone())) .collect(), ), ); } if let Some(relay_runtime) = relay_runtime { metadata.insert( "relay_config_path".to_string(), Value::String(relay_runtime.path.to_string_lossy().into_owned()), ); } Some(TelemetryRef { relay_enabled: telemetry.relay_enabled, provider: None, kind: None, uri: None, trace_id: None, metadata, }) } fn event_with_metadata( kind: impl Into, message: impl Into, metadata: BTreeMap, ) -> FabricEvent { FabricEvent { event_id: new_id("event"), timestamp_millis: now_millis(), kind: kind.into(), message: message.into(), metadata, } } pub(crate) fn new_id(prefix: &str) -> String { // The atomic counter only differentiates ids within one Fabric process. // Include the process id so independently running Fabric processes cannot // collide when they generate ids in the same millisecond. let counter = NEXT_ID.fetch_add(1, Ordering::Relaxed); format!("{prefix}-{}-{}-{counter}", now_millis(), std::process::id()) } fn now_millis() -> u128 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_millis()) .unwrap_or_default() } #[cfg(test)] mod tests { use std::fs; use std::io::Read; use std::net::TcpListener; use super::*; use crate::config::{ResolveContext, TelemetryProvider, resolve_run_plan_from_config}; fn local_host_plan(mode: &str) -> (PathBuf, RunPlan) { local_host_plan_with_relay(mode, false) } fn local_host_plan_with_relay(mode: &str, relay: bool) -> (PathBuf, RunPlan) { let root = std::env::temp_dir().join(new_id("fabric-local-host-test")); let _ = fs::remove_dir_all(&root); fs::create_dir_all(root.join("adapters/local-host")).expect("create adapters dir"); fs::write( root.join("adapters/local-host/local-host.fabric-adapter.json"), r#"{ "contract_version": "fabric.adapter/v1alpha2", "adapter_id": "acme.fabric.local-host", "adapter_kind": "python", "runner": {"module": "fake_host"}, "settings_schema": { "type": "object", "properties": { "python": {"type": "string"}, "cwd": {"type": "string"}, "env": { "type": "object", "additionalProperties": {"type": "string"} } }, "additionalProperties": false }, "extension_schemas": { "run_result": { "type": "object", "properties": {"trace_id": {"type": "string"}}, "additionalProperties": false }, "run_error": { "type": "object", "properties": {"source": {"type": "string"}}, "additionalProperties": false }, "usage": { "type": "object", "properties": { "provider": {"type": "string"}, "nemo_fabric.detailed_token_usage.v1": { "type": "object", "properties": { "input_tokens": {"type": "integer", "minimum": 0}, "cached_input_tokens": {"type": "integer", "minimum": 0}, "cache_write_input_tokens": {"type": "integer", "minimum": 0}, "input_tokens_include_cache": {"type": "boolean"}, "output_tokens": {"type": "integer", "minimum": 0}, "reasoning_tokens": {"type": "integer", "minimum": 0}, "output_tokens_include_reasoning": {"type": "boolean"}, "total_tokens": {"type": "integer", "minimum": 0}, "peak_request_input_tokens": {"type": "integer", "minimum": 0}, "models": { "type": "array", "items": { "type": "object", "properties": { "model": {"type": "string", "minLength": 1}, "provider": {"type": "string", "minLength": 1}, "input_tokens": {"type": "integer", "minimum": 0}, "reasoning_tokens": {"type": "integer", "minimum": 0} }, "required": ["model"], "additionalProperties": false } } }, "additionalProperties": false } }, "additionalProperties": false }, "artifact": { "type": "object", "properties": {"source": {"type": "string"}}, "additionalProperties": false } }, "telemetry": { "providers": { "relay": {"outputs": ["atif"]} } }, "capabilities": { "service": true } }"#, ) .expect("write adapter descriptor"); fs::write( root.join("fake_host.py"), r#"import json import os import socket import subprocess import sys import time MODE = os.environ.get("FABRIC_FAKE_HOST_MODE", "success") invocations = 0 def response(operation, *, output=None, error=None): outcome = ( {"status": "succeeded", "output": output} if error is None else {"status": "failed", "error": error} ) print(json.dumps({ "operation": operation, "outcome": outcome, }), flush=True) def failure(stage, code, message): return { "stage": stage, "code": code, "message": message, "retryable": False, } def read_http_response(stream): status = int(stream.readline().decode("ascii").split(" ", 2)[1]) while stream.readline() not in (b"\r\n", b"\n", b""): pass return status def write_openai_stream(sink, chunks): with socket.create_connection((sink["host"], sink["port"]), timeout=2) as client: request = ( "POST /openai-stream HTTP/1.1\r\n" f"Host: {sink['host']}:{sink['port']}\r\n" f"Authorization: Bearer {sink['token']}\r\n" "Content-Type: application/x-ndjson\r\n" "Transfer-Encoding: chunked\r\n" "Expect: 100-continue\r\n" "Connection: close\r\n\r\n" ) client.sendall(request.encode("ascii")) stream = client.makefile("rb") if read_http_response(stream) != 100: raise RuntimeError("stream listener rejected connection") records = [ { "type": "chunk", "sequence": index, "runtime_id": sink["runtime_id"], "invocation_id": sink["invocation_id"], "request_id": sink["request_id"], "chunk": chunk, } for index, chunk in enumerate(chunks) ] records.append({ "type": "end", "sequence": len(chunks), "runtime_id": sink["runtime_id"], "invocation_id": sink["invocation_id"], "request_id": sink["request_id"], }) for record in records: encoded = json.dumps(record, separators=(",", ":")).encode() + b"\n" client.sendall(f"{len(encoded):X}\r\n".encode() + encoded + b"\r\n") client.sendall(b"0\r\n\r\n") if read_http_response(stream) != 200: raise RuntimeError("stream listener rejected records") for line in sys.stdin: message = json.loads(line) operation = message["operation"] if operation == "start": if MODE == "start_failure": print("start diagnostic", file=sys.stderr, flush=True) response("start", error=failure("start", "fake_start", "start rejected")) sys.exit(16) if MODE == "invoke_timeout_exit": child = subprocess.Popen(["sleep", "30"]) root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: stream.write(str(child.pid)) if MODE == "child_after_start": subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(30)"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) service = message["payload"].get("service") if service and service["operation"] in {"prepare", "attach"}: with open(service["connection_path"], "w", encoding="utf-8") as stream: json.dump({"fake": "connection"}, stream) response("start", output={ "adapter_id": "acme.fabric.local-host", "service_type": "fake_service", "connection": {"endpoint": "http://127.0.0.1:1234"}, "metadata": {"ready": True}, }) else: response("start") if MODE == "crash_after_start": os.close(0) print("host crashed intentionally", file=sys.stderr, flush=True) time.sleep(1) sys.exit(17) elif operation in {"invoke", "invoke_openai_stream"}: invocations += 1 if MODE == "invoke_stderr": print(f"diagnostic-{invocations}", file=sys.stderr, flush=True) if MODE == "invoke_timeout": print("invoke accepted without response", file=sys.stderr, flush=True) continue if MODE == "invoke_timeout_exit": # Exit during Fabric's grace wait and leave the child running. time.sleep(1) os._exit(0) if MODE == "invoke_failure": response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue invocation = message["payload"] expected_fields = ( {"runtime_context", "request", "stream"} if operation == "invoke_openai_stream" else {"runtime_context", "request"} ) if set(invocation) != expected_fields: response(operation, error=failure( "invoke", "fake_invoke_shape", "invoke payload contains runtime config" )) continue if operation == "invoke_openai_stream": write_openai_stream(invocation["stream"], [ { "id": "chunk-1", "object": "chat.completion.chunk", "created": 0, "model": "test-model", "choices": [{"index": 0, "delta": {"content": "hel"}}], }, { "id": "chunk-2", "object": "chat.completion.chunk", "created": 0, "model": "test-model", "choices": [{"index": 0, "delta": {"content": "lo"}}], }, ]) if MODE == "memory_spike": subprocess.run([sys.executable, "-c", "import time\ndata=bytearray(192*1024*1024)\n" "for i in range(0,len(data),4096): data[i]=1\n" "time.sleep(0.7)"], check=True) if MODE == "busy": # A descendant that burns 1.2 s of CPU, then exits and is waited for. subprocess.run([sys.executable, "-c", "import time\nwhile time.process_time() < 1.2:\n pass"], check=True) output = { "host_pid": os.getpid(), "invocation_count": invocations, "input": invocation["request"]["input"], "runtime_id": invocation["runtime_context"]["runtime_id"], "invocation_id": invocation["runtime_context"]["invocation_id"], "request_id": invocation["runtime_context"]["request_id"], "normalized_env": os.environ.get("FABRIC_NORMALIZED_ENV"), "context_env": invocation["runtime_context"]["environment"].get("env", {}), "deadline_millis": invocation["runtime_context"].get("deadline_millis"), "state_dir": invocation["runtime_context"].get("state_dir"), } if MODE in {"adapter_reported_failure", "blank_error_code"}: result = { "status": "failed", "output": output, "error": { "code": " \t" if MODE == "blank_error_code" else "fake_adapter_failure", "message": "adapter rejected the invocation", "retryable": True, "extensions": {"source": "fake-host"}, }, } elif MODE == "typed_result": result = { "status": "succeeded", "output": output, "usage": { "input_tokens": 3, "cached_input_tokens": 1, "input_tokens_include_cache": True, "output_tokens": 5, "total_tokens": 8, "cost_usd": 0.25, "extensions": {"provider": "fake"}, }, "artifacts": [{ "name": "report", "kind": "report", "path": "reports/result.json", "media_type": "application/json", "extensions": {"source": "fake-host"}, }], "extensions": {"trace_id": "trace-1"}, } elif MODE == "detailed_usage_result": result = { "status": "succeeded", "output": output, "usage": { "input_tokens": 3, "cached_input_tokens": 1, "input_tokens_include_cache": True, "output_tokens": 5, "total_tokens": 8, "cost_usd": 0.25, "extensions": { "provider": "fake", "nemo_fabric.detailed_token_usage.v1": { "input_tokens": 20, "cached_input_tokens": 5, "cache_write_input_tokens": 3, "input_tokens_include_cache": False, "output_tokens": 5, "reasoning_tokens": 4, "output_tokens_include_reasoning": False, "total_tokens": 37, "peak_request_input_tokens": 18, "models": [{ "model": "test-model", "provider": "fake", "input_tokens": 20, "reasoning_tokens": 4, }], }, }, }, } else: result = {"status": "succeeded", "output": output} response(operation, output=result) elif operation == "stop": if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") time.sleep(0.25) if MODE == "stop_failure": response("stop", error=failure("stop", "fake_stop", "stop rejected")) sys.exit(18) response("stop") break else: if MODE == "linger_after_exit": # Outlive the process that started this host, as a host still # stopping its harness after that process crashed would. root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) with open(os.path.join(root, "host.pid.tmp"), "w", encoding="utf-8") as stream: stream.write(str(os.getpid())) os.replace(os.path.join(root, "host.pid.tmp"), os.path.join(root, "host.pid")) time.sleep(30) "#, ) .expect("write fake host"); let mut config_value = serde_json::json!({ "schema_version": "fabric.agent/v1alpha1", "metadata": {"name": "local-host-test-agent"}, "harness": { "adapter_id": "acme.fabric.local-host", "resolution": "preinstalled", "settings": { "python": "python3", "cwd": ".", "env": { "FABRIC_FAKE_HOST_MODE": mode, "FABRIC_FAKE_HOST_STOP_MARKER": root.join("stop-started"), }, }, }, "discovery": {"local_paths": ["adapters"]}, "runtime": { "input_schema": "text", "output_schema": "text", "artifacts": "./artifacts", }, "environment": { "provider": "local", "env": {"FABRIC_NORMALIZED_ENV": "visible"}, }, }); if relay { config_value["telemetry"] = serde_json::json!({ "providers": {"relay": {}}, }); config_value["relay"] = serde_json::json!({ "observability": {"atif": {"enabled": true}}, }); } let config: crate::config::FabricConfig = serde_json::from_value(config_value).expect("typed config"); let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) .expect("resolve local-host plan"); (root, plan) } fn openai_stream_listener( token: &str, ) -> (OpenAiStreamTransport, thread::JoinHandle>) { let listener = TcpListener::bind((OPENAI_STREAM_HOST, 0)).expect("bind stream listener"); let port = listener.local_addr().expect("listener address").port(); let expected_token = token.to_string(); let capture = thread::spawn(move || { listener .set_nonblocking(true) .expect("set stream listener nonblocking"); let accept_deadline = Instant::now() + Duration::from_secs(5); let mut stream = loop { match listener.accept() { Ok((stream, _)) => break stream, Err(error) if error.kind() == ErrorKind::WouldBlock && Instant::now() < accept_deadline => { thread::sleep(Duration::from_millis(10)); } Err(error) if error.kind() == ErrorKind::WouldBlock => { panic!("timed out waiting for stream connection") } Err(error) => panic!("accept stream connection: {error}"), } }; stream .set_nonblocking(false) .expect("set accepted stream blocking"); stream .set_read_timeout(Some(Duration::from_secs(5))) .expect("set stream read timeout"); let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); let mut headers = Vec::new(); loop { let mut line = String::new(); reader.read_line(&mut line).expect("read HTTP header"); if line == "\r\n" { break; } headers.push(line); } assert_eq!( headers.first().map(String::as_str), Some("POST /openai-stream HTTP/1.1\r\n") ); assert!(headers.iter().any(|header| { header == &format!("Authorization: Bearer {expected_token}\r\n") })); stream .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") .expect("accept stream request"); stream.flush().expect("flush continue response"); let mut records = Vec::new(); loop { let mut size_line = String::new(); reader.read_line(&mut size_line).expect("read chunk size"); let size = usize::from_str_radix(size_line.trim(), 16).expect("hex chunk size"); if size == 0 { let mut terminator = String::new(); reader .read_line(&mut terminator) .expect("read chunk terminator"); assert_eq!(terminator, "\r\n"); break; } let mut encoded = vec![0; size]; reader.read_exact(&mut encoded).expect("read chunk body"); let mut terminator = [0; 2]; reader .read_exact(&mut terminator) .expect("read chunk terminator"); assert_eq!(&terminator, b"\r\n"); records.push( serde_json::from_slice(encoded.strip_suffix(b"\n").unwrap_or(&encoded)) .expect("parse stream record"), ); } stream .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") .expect("complete stream request"); stream.flush().expect("flush final response"); records }); ( OpenAiStreamTransport { port, token: token.to_string(), }, capture, ) } #[test] fn adapter_lifecycle_always_receives_southbound_agent_config() { let (root, plan) = local_host_plan("success"); let southbound = serde_json::to_value(adapter_lifecycle_config(&plan)) .expect("serialize southbound adapter config"); assert!(southbound.get("schema_version").is_none()); assert!(southbound.get("metadata").is_none()); assert_eq!( southbound["harness"]["settings"]["python"], serde_json::json!("python3") ); let _ = fs::remove_dir_all(root); } #[test] fn runtime_context_preserves_native_telemetry_config() { let (root, mut plan) = local_host_plan("success"); let native_config = serde_json::json!({"components": [{"kind": "observability"}]}); plan.telemetry_plan = Some(TelemetryPlan { providers: vec![TelemetryProvider::Native], relay_enabled: false, relay_project: None, relay_output_dir: None, relay_config: None, native_config: Some(native_config.clone()), adapter_outputs: Vec::new(), }); let telemetry = runtime_telemetry_context(&plan, None).expect("telemetry context"); assert_eq!(telemetry.metadata["native_config"], native_config); let _ = fs::remove_dir_all(root); } #[test] fn sdk_run_result_preserves_ordered_telemetry_references() { let result = RunResult { agent_name: "agent".to_string(), harness: "harness".to_string(), adapter_kind: AdapterKind::Python, adapter_id: None, runtime_id: "runtime-1".to_string(), invocation_id: "invocation-1".to_string(), request_id: "request-1".to_string(), status: RunStatus::Succeeded, output: Value::Null, error: None, usage: None, execution: None, resource_snapshot: None, system_stats: None, artifacts: ArtifactManifest::default(), telemetry: vec![ TelemetryRef { relay_enabled: true, provider: Some("relay".to_string()), kind: Some("trace".to_string()), uri: Some("artifacts/trace".to_string()), trace_id: Some("trace-1".to_string()), metadata: BTreeMap::new(), }, TelemetryRef { relay_enabled: false, provider: Some("native".to_string()), kind: Some("metrics".to_string()), uri: Some("artifacts/metrics".to_string()), trace_id: None, metadata: BTreeMap::new(), }, ], events: Vec::new(), metadata: BTreeMap::new(), }; let encoded = serde_json::to_value(&result).expect("serialize SDK run result"); assert!(encoded["telemetry"].is_array()); let decoded: RunResult = serde_json::from_value(encoded).expect("deserialize SDK run result"); assert_eq!(decoded.telemetry, result.telemetry); } fn stopped_agents() -> Vec { TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } #[cfg(windows)] #[test] fn windows_busy_stop_retains_ownership_without_a_job() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); let control = windows_host_controls() .get(&runtime.runtime_id) .unwrap() .clone(); let process_only = Arc::new(WindowsHostControl { process: control.process.try_clone().unwrap(), job: None, }); windows_host_controls().insert(runtime.runtime_id.clone(), process_only.clone()); let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); let guard = host.lock().unwrap(); let stopped = stop_runtime(&plan, &runtime); drop(guard); let retained = windows_host_controls().contains_key(&runtime.runtime_id); let still_running = process_only.has_exited(); windows_host_controls().insert(runtime.runtime_id.clone(), control); stop_runtime(&plan, &runtime).unwrap(); assert!( matches!(stopped, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "cleanup_unconfirmed") ); assert!(retained); assert_eq!(still_running, Some(false)); assert_eq!(process_only.cleanup_confirmed(), None); fs::remove_dir_all(root).unwrap(); } #[cfg(windows)] #[test] fn windows_stop_confirms_the_job_after_the_host_exits() { let (root, plan) = local_host_plan("child_after_start"); let runtime = start_runtime(&plan).unwrap(); let control = windows_host_controls() .get(&runtime.runtime_id) .unwrap() .clone(); assert_eq!(control.has_exited(), Some(false)); { let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); let mut host = host.lock().unwrap(); host.child.kill().unwrap(); host.child.wait().unwrap(); } assert_eq!(control.has_exited(), Some(true)); assert_eq!(control.cleanup_confirmed(), Some(false)); stop_runtime(&plan, &runtime).expect("stop children after the host already exited"); assert_eq!(control.cleanup_confirmed(), Some(true)); assert!(!windows_host_controls().contains_key(&runtime.runtime_id)); fs::remove_dir_all(root).unwrap(); } #[test] fn busy_stop_interrupts_the_owned_host_without_the_invocation_lock() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); let guard = host.lock().unwrap(); let interrupted = interrupt_local_host(&runtime); drop(guard); let stopped = stop_runtime(&plan, &runtime); interrupted.expect("interrupt without taking the busy host lock"); stopped.expect("confirm cleanup after interruption"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!(local_host_has_exited(&runtime.runtime_id).is_none()); fs::remove_dir_all(root).unwrap(); } #[cfg(unix)] #[test] fn stop_retries_can_confirm_a_child_that_exited_before_identity_capture() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); let mut child = Command::new("sh").args(["-c", "exit 0"]).spawn().unwrap(); let pid = child.id(); child.wait().unwrap(); let gone = LocalHostProcess::new(pid, SystemTime::now()); assert!(!gone.is_current()); local_hosts() .get(&runtime.runtime_id) .unwrap() .lock() .unwrap() .pending_stop .push(gone); stop_runtime(&plan, &runtime) .expect("a confirmed-gone child cannot permanently block cleanup"); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!(local_host_has_exited(&runtime.runtime_id).is_none()); fs::remove_dir_all(root).unwrap(); } #[test] fn local_host_reuses_one_process_and_stops_idempotently() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); let first = invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); let second = invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); assert_eq!(first.output["host_pid"], second.output["host_pid"]); assert_eq!(first.output["invocation_count"], serde_json::json!(1)); assert_eq!(second.output["invocation_count"], serde_json::json!(2)); assert_eq!(first.output["input"], serde_json::json!("first")); assert_eq!(first.output["normalized_env"], serde_json::json!("visible")); assert_eq!(second.output["input"], serde_json::json!("second")); assert_eq!(first.metadata["host_pid"], second.metadata["host_pid"]); let persisted_invocation: Value = serde_json::from_str( &fs::read_to_string( first.metadata["fabric_invocation"] .as_str() .expect("invocation path"), ) .expect("read persisted invocation"), ) .expect("parse persisted invocation"); assert_eq!( persisted_invocation["runtime_context"]["environment"]["env"]["FABRIC_NORMALIZED_ENV"], serde_json::json!("[REDACTED]") ); assert_eq!( first.metadata["adapter_runner"], serde_json::json!("persistent_local_host") ); let stdout = first .artifacts .artifacts .iter() .find(|artifact| artifact.name == "stdout") .expect("stdout artifact"); let captured: Value = serde_json::from_str(&fs::read_to_string(&stdout.path).expect("read stdout artifact")) .expect("parse stdout artifact"); assert_eq!(captured, first.output); assert!( first .artifacts .artifacts .iter() .all(|artifact| artifact.name != "stderr") ); let first_stop = stop_runtime(&plan, &runtime).expect("first stop"); let second_stop = stop_runtime(&plan, &runtime).expect("idempotent stop"); assert_eq!(first_stop[0].metadata["already_stopped"], false); assert_eq!(second_stop[0].metadata["already_stopped"], true); let _ = fs::remove_dir_all(root); } fn local_host_plan_keeping_state() -> (PathBuf, RunPlan) { local_host_plan_keeping_state_in("success") } fn local_host_plan_keeping_state_in(mode: &str) -> (PathBuf, RunPlan) { let (root, plan) = local_host_plan(mode); let descriptor_path = root.join("adapters/local-host/local-host.fabric-adapter.json"); let mut descriptor: Value = serde_json::from_str( &fs::read_to_string(&descriptor_path).expect("read adapter descriptor"), ) .expect("parse adapter descriptor"); descriptor["config"] = serde_json::json!({"accepts": ["runtime.keep_state"]}); fs::write(&descriptor_path, descriptor.to_string()).expect("write adapter descriptor"); let mut config = plan.config; config.metadata.name = "assistant".to_string(); config.runtime.keep_state = true; let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) .expect("resolve keep_state plan"); (root, plan) } #[test] fn keep_state_gives_an_agents_runtimes_one_state_folder_one_at_a_time() { let (root, plan) = local_host_plan_keeping_state(); let state_dir = |runtime: &RuntimeHandle| { invoke_runtime(&plan, runtime, RunRequest::text("state")) .expect("invocation") .output["state_dir"] .clone() }; let first = start_runtime(&plan).expect("start first runtime"); let error = start_runtime(&plan).expect_err("a second runtime must not share the folder"); assert!(matches!(error, FabricError::StateInUse { .. }), "{error}"); let first_dir = state_dir(&first); stop_runtime(&plan, &first).expect("stop first runtime"); let second = start_runtime(&plan).expect("start once the first runtime stopped"); let second_dir = state_dir(&second); stop_runtime(&plan, &second).expect("stop second runtime"); let expected = root.join("artifacts/.fabric/state/acme.fabric.local-host/assistant"); assert_ne!(first.runtime_id, second.runtime_id); assert_eq!(second_dir, first_dir); let first_dir = first_dir.as_str().expect("state_dir in runtime context"); assert!(Path::new(first_dir).is_absolute(), "{first_dir}"); assert_eq!(Path::new(first_dir), expected); assert!(expected.is_dir()); // The lock sits beside the folder, so an adapter that replaces the // folder can't drop it. assert!(!expected.join(".fabric-state.lock").exists()); assert!(expected.with_file_name(".assistant.lock").is_file()); let _ = fs::remove_dir_all(root); } #[test] fn workspace_patches_leave_out_kept_agent_state_by_literal_path() { for tracked in [false, true] { let root = std::env::temp_dir().join(new_id("fabric-workspace-patch-test")); // A bracket in the folder name must not act as a pathspec pattern, // which would also leave out "artifacts1/.fabric/state". let kept = root.join("artifacts[1]/.fabric/state"); let state = kept.join("acme.agent/assistant"); fs::create_dir_all(&state).expect("create state folder"); fs::write(state.join("state.db"), "private\n").expect("write state file"); fs::create_dir_all(root.join("artifacts1/.fabric")).expect("create lookalike folder"); fs::write(root.join("artifacts1/.fabric/state"), "workspace\n") .expect("write lookalike"); fs::write(root.join("notes.txt"), "note\n").expect("write workspace file"); let git = |args: &[&str]| { let status = Command::new("git").arg("-C").arg(&root).args(args).status(); assert!(status.expect("run git").success()); }; git(&["init", "-q"]); let patch = if tracked { git(&["add", "--intent-to-add", "--", "."]); tracked_workspace_patch(&root, std::slice::from_ref(&kept)).expect("tracked patch") } else { untracked_workspace_patch(&root, std::slice::from_ref(&kept)) .expect("untracked patch") }; assert!(patch.contains("notes.txt"), "{patch}"); assert!(patch.contains("artifacts1/.fabric/state"), "{patch}"); assert!(!patch.contains("state.db"), "{patch}"); let _ = fs::remove_dir_all(root); } } #[test] fn keep_state_planning_rejects_unaccepting_adapters_and_unsafe_agent_names() { let (root, accepting) = local_host_plan_keeping_state(); let (plain_root, plain) = local_host_plan("success"); let mut not_accepted = plain.config; not_accepted.runtime.keep_state = true; let mut cases = vec![(&plain_root, not_accepted, "runtime.keep_state")]; let mut without_artifacts = accepting.config.clone(); without_artifacts.runtime.artifacts = None; cases.push((&root, without_artifacts, "runtime.keep_state")); for name in [ "..", "a/b", "a\\b", "a:b", "a*b", "a?b", "a|b", "ab", "a\"b", "a\u{1}b", "name.", "name ", "CON", "nul", "Aux.txt", "COM1", "lpt9", "COM\u{b9}", ] { let mut config = accepting.config.clone(); config.metadata.name = name.to_string(); cases.push((&root, config, "metadata.name")); } for (base_dir, config, field) in cases { let error = resolve_run_plan_from_config(config, ResolveContext::new(base_dir)) .expect_err(field); assert!(error.to_string().contains(field), "{field}: {error}"); } let _ = fs::remove_dir_all(root); let _ = fs::remove_dir_all(plain_root); } #[test] fn keep_state_rejects_a_run_plan_whose_names_leave_the_state_folder() { for (field, value) in [ ("agent_name", "../../outside"), ("adapter_id", "../../outside"), ("adapter_id", "/outside"), ("adapter_id", "CON"), ] { let (root, mut plan) = local_host_plan_keeping_state(); if field == "agent_name" { plan.agent_name = value.to_string(); } else if let Some(adapter) = plan.adapter_descriptor.as_mut() { adapter.descriptor.adapter_id = value.to_string(); } let error = start_runtime(&plan).expect_err(value); assert!( matches!(&error, FabricError::InvalidConfig { field: name, .. } if name == field), "{field} {value}: {error}" ); assert!(!root.join("artifacts/.fabric/outside").exists(), "{value}"); assert!(!root.join("artifacts/outside").exists(), "{value}"); assert!(!Path::new("/outside").exists(), "{value}"); let _ = fs::remove_dir_all(root); } } #[test] fn one_folder_name_accepts_names_that_are_folders_on_every_platform() { for name in [ "assistant", "my agent", "a.b", ".hidden", "CONSOLE", "COM10", ] { assert!(is_one_folder_name(name), "{name}"); } } #[cfg(unix)] #[test] fn keep_state_creates_the_state_folder_for_its_owner_only() { use std::os::unix::fs::PermissionsExt; let (root, plan) = local_host_plan_keeping_state(); let state_dir = root.join("artifacts/.fabric/state/acme.fabric.local-host/assistant"); let mode = |path: &Path| { fs::metadata(path) .expect("state folder") .permissions() .mode() & 0o777 }; let first = start_runtime(&plan).expect("start runtime"); assert_eq!(mode(&state_dir), 0o700); stop_runtime(&plan, &first).expect("stop runtime"); fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o755)).expect("widen"); let second = start_runtime(&plan).expect("start runtime again"); assert_eq!(mode(&state_dir), 0o700, "an existing folder is narrowed"); stop_runtime(&plan, &second).expect("stop runtime again"); let _ = fs::remove_dir_all(root); } #[cfg(unix)] #[test] fn keep_state_stays_locked_while_a_host_outlives_the_process_that_started_it() { const APP_PLAN: &str = "FABRIC_KEEP_STATE_TEST_APP_PLAN"; if let Ok(path) = std::env::var(APP_PLAN) { // The app: start a runtime, then exit without stopping it. let plan: RunPlan = serde_json::from_str(&fs::read_to_string(path).expect("read plan")).expect("plan"); start_runtime(&plan).expect("start runtime in the app"); std::process::exit(0); } let (root, plan) = local_host_plan_keeping_state_in("linger_after_exit"); let plan_path = root.join("plan.json"); fs::write(&plan_path, serde_json::to_string(&plan).expect("plan JSON")) .expect("write plan"); let app = Command::new(std::env::current_exe().expect("test binary")) .args([ "runtime::tests::keep_state_stays_locked_while_a_host_outlives_the_process_that_started_it", "--exact", "--nocapture", ]) .env(APP_PLAN, &plan_path) .status() .expect("run the app"); assert!(app.success(), "{app}"); let pid_path = root.join("host.pid"); let deadline = Instant::now() + Duration::from_secs(10); while !pid_path.exists() && Instant::now() < deadline { thread::sleep(Duration::from_millis(20)); } let host: libc::pid_t = fs::read_to_string(&pid_path) .expect("the host outlives the app") .trim() .parse() .expect("host pid"); let second = start_runtime(&plan); // SAFETY: kill has no memory-safety preconditions. unsafe { libc::kill(host, libc::SIGKILL) }; let error = second.expect_err("the surviving host still uses the state"); assert!(matches!(error, FabricError::StateInUse { .. }), "{error}"); let deadline = Instant::now() + Duration::from_secs(5); let runtime = loop { match start_runtime(&plan) { Ok(runtime) => break runtime, Err(FabricError::StateInUse { .. }) if Instant::now() < deadline => { thread::sleep(Duration::from_millis(20)); } Err(error) => panic!("start after the host exited: {error}"), } }; stop_runtime(&plan, &runtime).expect("stop runtime"); let _ = fs::remove_dir_all(root); } #[test] fn keep_state_lets_a_service_and_one_connected_runtime_use_the_state_folder() { let (root, plan) = local_host_plan_keeping_state(); let service = prepare_service(&plan).expect("prepare service"); let first = start_runtime_with_service(&plan, &service).expect("first connected runtime"); let error = start_runtime_with_service(&plan, &service) .expect_err("a second connected runtime must not share the folder"); assert!(matches!(error, FabricError::StateInUse { .. }), "{error}"); let error = start_runtime(&plan).expect_err("an independent runtime must not share it"); assert!(matches!(error, FabricError::StateInUse { .. }), "{error}"); stop_runtime(&plan, &first).expect("stop first runtime"); let second = start_runtime_with_service(&plan, &service) .expect("connect once the first runtime stopped"); stop_runtime(&plan, &second).expect("stop second runtime"); release_service(&plan, &service).expect("release service"); let independent = start_runtime(&plan).expect("start once the service is released"); stop_runtime(&plan, &independent).expect("stop independent runtime"); let _ = fs::remove_dir_all(root); } #[cfg(unix)] #[test] fn keep_state_stays_locked_while_a_connected_host_outlives_its_service() { const APP_PLAN: &str = "FABRIC_KEEP_STATE_SERVICE_TEST_PLAN"; if let Ok(path) = std::env::var(APP_PLAN) { let plan: RunPlan = serde_json::from_str(&fs::read_to_string(path).expect("read plan")).expect("plan"); let service = prepare_service(&plan).expect("prepare service in the app"); start_runtime_with_service(&plan, &service).expect("connect runtime in the app"); // The service can exit before a connected host finishes cleanup. let record = local_services().get(&service.service_id).cloned().unwrap(); let mut record = record.lock().unwrap_or_else(|error| error.into_inner()); record.host.child.kill().expect("kill service host"); record.host.child.wait().expect("wait for service host"); std::process::exit(0); } let (root, plan) = local_host_plan_keeping_state_in("linger_after_exit"); let plan_path = root.join("plan.json"); fs::write(&plan_path, serde_json::to_string(&plan).expect("plan JSON")) .expect("write plan"); let app = Command::new(std::env::current_exe().expect("test binary")) .args([ "runtime::tests::keep_state_stays_locked_while_a_connected_host_outlives_its_service", "--exact", "--nocapture", ]) .env(APP_PLAN, &plan_path) .status() .expect("run the app"); assert!(app.success(), "{app}"); let pid_path = root.join("host.pid"); let deadline = Instant::now() + Duration::from_secs(10); while !pid_path.exists() && Instant::now() < deadline { thread::sleep(Duration::from_millis(20)); } let host: libc::pid_t = fs::read_to_string(&pid_path) .expect("the connected host outlives the app and service") .trim() .parse() .expect("connected host pid"); let second = start_runtime(&plan); if let Ok(runtime) = &second { stop_runtime(&plan, runtime).expect("stop overlapping runtime"); } // SAFETY: the test owns this host, which remains alive until signalled. unsafe { libc::kill(host, libc::SIGKILL) }; let error = second.expect_err("the connected host still uses the state"); assert!(matches!(error, FabricError::StateInUse { .. }), "{error}"); let deadline = Instant::now() + Duration::from_secs(5); let runtime = loop { match start_runtime(&plan) { Ok(runtime) => break runtime, Err(FabricError::StateInUse { .. }) if Instant::now() < deadline => { thread::sleep(Duration::from_millis(20)); } Err(error) => panic!("start after the connected host exited: {error}"), } }; stop_runtime(&plan, &runtime).expect("stop replacement runtime"); let _ = fs::remove_dir_all(root); } #[test] fn keep_state_lets_a_runtime_connect_after_another_failed_to_stop() { let (root, plan) = local_host_plan_keeping_state_in("stop_failure"); let service = prepare_service(&plan).expect("prepare service"); let first = start_runtime_with_service(&plan, &service).expect("first connected runtime"); stop_runtime(&plan, &first).expect_err("the adapter fails its stop"); let second = start_runtime_with_service(&plan, &service) .expect("connect once the first runtime's host is gone"); stop_runtime(&plan, &second).expect_err("the adapter fails its stop"); assert!(matches!( release_service(&plan, &service), Err(FabricError::AdapterLifecycleOperation { .. }) )); release_service(&plan, &service).expect("retry release service"); assert!(!local_services().contains_key(&service.service_id)); let _ = fs::remove_dir_all(root); } #[test] fn keep_state_rejects_service_connection_after_relative_base_changes() { const APP_PLAN: &str = "FABRIC_KEEP_STATE_RELATIVE_PLAN"; if let Ok(path) = std::env::var(APP_PLAN) { let mut plan: RunPlan = serde_json::from_str(&fs::read_to_string(path).expect("read plan")).expect("plan"); plan.base_dir = PathBuf::from("."); let original = std::env::current_dir().expect("original directory"); plan.config .harness .as_mut() .expect("harness") .settings .insert("cwd".to_string(), serde_json::json!(original)); let service = prepare_service(&plan).expect("prepare service with relative base"); let runtime = start_runtime_with_service(&plan, &service) .expect("connect from the original directory"); stop_runtime(&plan, &runtime).expect("stop connected runtime"); std::env::set_current_dir("other").expect("change directory in isolated app"); let second = start_runtime_with_service(&plan, &service); if let Ok(runtime) = &second { stop_runtime(&plan, runtime).expect("stop unexpected connected runtime"); } std::env::set_current_dir(original).expect("restore app directory"); release_service(&plan, &service).expect("release service"); assert!( matches!(second, Err(FabricError::ServiceHandleMismatch { .. })), "a service connection must remain bound to its original directory: {second:?}" ); std::process::exit(0); } let (root, plan) = local_host_plan_keeping_state(); fs::create_dir(root.join("other")).expect("create other directory"); let plan_path = root.join("plan.json"); fs::write(&plan_path, serde_json::to_string(&plan).expect("plan JSON")) .expect("write plan"); let app = Command::new(std::env::current_exe().expect("test binary")) .args([ "runtime::tests::keep_state_rejects_service_connection_after_relative_base_changes", "--exact", "--nocapture", ]) .current_dir(&root) .env(APP_PLAN, &plan_path) .status() .expect("run isolated app"); let _ = fs::remove_dir_all(root); assert!(app.success(), "{app}"); } fn local_host_plan_with_env_name(name: &str) -> (PathBuf, RunPlan) { let (root, plan) = local_host_plan("success"); let mut config = plan.config; config.environment.as_mut().expect("environment").env_names = vec![name.to_string()]; let plan = resolve_run_plan_from_config(config, ResolveContext::new(&root)) .expect("resolve local-host plan with env_names"); (root, plan) } #[test] fn env_names_values_reach_only_the_adapter_runtime_context() { let (root, plan) = local_host_plan_with_env_name("PATH"); let value = std::env::var("PATH").expect("test process PATH"); let runtime = start_runtime(&plan).expect("start local host"); let result = invoke_runtime(&plan, &runtime, RunRequest::text("named")).expect("invocation"); stop_runtime(&plan, &runtime).expect("stop"); assert_eq!( result.output["context_env"]["PATH"], serde_json::json!(value) ); assert_eq!( result.output["context_env"]["FABRIC_NORMALIZED_ENV"], serde_json::json!("visible") ); assert_eq!(result.output["state_dir"], Value::Null); let persisted_invocation: Value = serde_json::from_str( &fs::read_to_string( result.metadata["fabric_invocation"] .as_str() .expect("invocation path"), ) .expect("read persisted invocation"), ) .expect("parse persisted invocation"); assert_eq!( persisted_invocation["runtime_context"]["environment"]["env"]["PATH"], serde_json::json!("[REDACTED]") ); let plan_json = serde_json::to_string(&plan).expect("serialize plan"); assert!(!plan_json.contains(&value)); assert!(!runtime.environment.env.contains_key("PATH")); let _ = fs::remove_dir_all(root); } #[test] fn env_names_fail_runtime_start_when_a_variable_is_unset() { let (root, plan) = local_host_plan_with_env_name("NEMO_FABRIC_TEST_UNSET_VARIABLE"); let error = start_runtime(&plan).expect_err("unset variable must fail"); assert!(matches!( &error, FabricError::InvalidConfig { field, reason } if field == "environment.env_names" && reason == "environment variable `NEMO_FABRIC_TEST_UNSET_VARIABLE` is not set" )); let _ = fs::remove_dir_all(root); } #[test] fn env_names_reject_runtime_connection_to_a_service_with_different_values() { let (root, plan) = local_host_plan_with_env_name("PATH"); let service = prepare_service(&plan).expect("prepare service"); let record = local_services() .get(&service.service_id) .cloned() .expect("service record"); // Simulate a service started with a previous value without mutating // the process environment while other Rust tests are running. let original = record .lock() .expect("service lock") .host .environment_env .insert("PATH".to_string(), "previous-service-value".to_string()) .expect("startup PATH"); let error = start_runtime_with_service(&plan, &service) .expect_err("changed service environment must fail"); assert!(matches!( &error, FabricError::InvalidConfig { field, reason } if field == "environment.env_names" && reason.contains("environment variable `PATH` changed") && reason.contains("restart the runtime and any associated service") )); assert!(!error.to_string().contains("previous-service-value")); record .lock() .expect("service lock") .host .environment_env .insert("PATH".to_string(), original); let runtime = start_runtime_with_service(&plan, &service).expect("unchanged environment"); stop_runtime(&plan, &runtime).expect("stop runtime"); release_service(&plan, &service).expect("release service"); let _ = fs::remove_dir_all(root); } #[test] fn service_hosts_share_across_runtimes_and_require_ordered_release() { let (root, plan) = local_host_plan("success"); let service = prepare_service(&plan).expect("prepare service"); let first = start_runtime_with_service(&plan, &service).expect("first runtime"); let second = start_runtime_with_service(&plan, &service).expect("second runtime"); assert_eq!( first.service_id.as_deref(), Some(service.service_id.as_str()) ); assert_eq!( second.service_id.as_deref(), Some(service.service_id.as_str()) ); assert_ne!(first.runtime_id, second.runtime_id); let mut modified_service = service.clone(); modified_service.connection.insert( "endpoint".to_string(), serde_json::json!("http://127.0.0.1:9999"), ); assert!(matches!( start_runtime_with_service(&plan, &modified_service), Err(FabricError::ServiceHandleMismatch { .. }) )); assert!(matches!( release_service(&plan, &service), Err(FabricError::ServiceInUse { .. }) )); let first_result = invoke_runtime(&plan, &first, RunRequest::text("first")).expect("invoke first runtime"); let second_result = invoke_runtime(&plan, &second, RunRequest::text("second")) .expect("invoke second runtime"); assert_ne!( first_result.output["host_pid"], second_result.output["host_pid"] ); stop_runtime(&plan, &first).expect("stop first runtime"); stop_runtime(&plan, &second).expect("stop second runtime"); let released = release_service(&plan, &service).expect("release service"); let repeated = release_service(&plan, &service).expect("idempotent release"); assert_eq!(released[0].metadata["already_released"], false); assert_eq!(repeated[0].metadata["already_released"], true); assert!(matches!( start_runtime_with_service(&plan, &service), Err(FabricError::ServiceNotActive { .. }) )); let attached = attach_service( &plan, ServiceReference { adapter_id: "acme.fabric.local-host".to_string(), service_type: "fake_service".to_string(), connection: BTreeMap::from([( "token_env".to_string(), serde_json::json!("FAKE_SERVICE_TOKEN"), )]), metadata: BTreeMap::new(), }, ) .expect("attach service"); assert_eq!(attached.ownership, ServiceOwnership::CallerOwned); release_service(&plan, &attached).expect("detach service"); let _ = fs::remove_dir_all(root); } #[test] fn service_release_blocks_new_runtime_startup() { let (root, plan) = local_host_plan("stop_delay"); let service = prepare_service(&plan).expect("prepare service"); let release_plan = plan.clone(); let release_handle = service.clone(); let release = thread::spawn(move || release_service(&release_plan, &release_handle)); let marker = root.join("stop-started"); let deadline = Instant::now() + Duration::from_secs(5); while !marker.exists() && Instant::now() < deadline { thread::sleep(Duration::from_millis(10)); } assert!(marker.exists(), "service stop did not begin"); assert!(matches!( start_runtime_with_service(&plan, &service), Err(FabricError::ServiceNotActive { .. }) )); release .join() .expect("release thread") .expect("release service"); let _ = fs::remove_dir_all(root); } #[test] fn failed_service_release_retains_record_for_retry() { let (root, plan) = local_host_plan("stop_failure"); let service = prepare_service(&plan).expect("prepare service"); assert!(matches!( release_service(&plan, &service), Err(FabricError::AdapterLifecycleOperation { .. }) )); assert!(local_services().contains_key(&service.service_id)); assert!(matches!( start_runtime_with_service(&plan, &service), Err(FabricError::ServiceNotActive { .. }) )); let released = release_service(&plan, &service).expect("retry release service"); assert_eq!(released[0].metadata["already_released"], false); assert!(!local_services().contains_key(&service.service_id)); let _ = fs::remove_dir_all(root); } #[test] fn local_host_openai_stream_uses_side_channel_and_separate_terminal_result() { let (root, mut plan) = local_host_plan("success"); plan.capabilities.streaming = true; plan.adapter_descriptor .as_mut() .expect("resolved descriptor") .descriptor .capabilities .streaming = true; let runtime = start_runtime(&plan).expect("start local host"); let (transport, capture) = openai_stream_listener("stream-secret"); let result = invoke_openai_stream(&plan, &runtime, RunRequest::text("stream"), transport) .expect("stream invocation"); let records = capture.join().expect("stream capture"); assert_eq!(records.len(), 3); assert_eq!(records[0]["type"], "chunk"); assert_eq!(records[0]["sequence"], 0); assert_eq!(records[0]["chunk"]["id"], "chunk-1"); assert_eq!(records[1]["chunk"]["id"], "chunk-2"); assert_eq!(records[2]["type"], "end"); assert_eq!(records[2]["sequence"], 2); for record in &records { assert_eq!(record["runtime_id"], result.runtime_id); assert_eq!(record["invocation_id"], result.invocation_id); assert_eq!(record["request_id"], result.request_id); } assert_eq!(result.output["input"], "stream"); let persisted = fs::read_to_string( result.metadata["fabric_invocation"] .as_str() .expect("invocation path"), ) .expect("read persisted invocation"); assert!(!persisted.contains("stream-secret")); let persisted: Value = serde_json::from_str(&persisted).expect("parse invocation"); assert_eq!(persisted["stream"]["token"], "[REDACTED]"); assert_eq!( persisted["runtime_context"]["environment"]["env"]["FABRIC_NORMALIZED_ENV"], "[REDACTED]" ); assert_eq!( persisted["stream"]["protocol_version"], OPENAI_STREAM_PROTOCOL_VERSION ); assert_eq!( persisted["stream"]["profile"], OPENAI_CHAT_COMPLETIONS_CHUNK_PROFILE ); assert_eq!(persisted["stream"]["runtime_id"], result.runtime_id); assert_eq!(persisted["stream"]["invocation_id"], result.invocation_id); assert_eq!(persisted["stream"]["request_id"], result.request_id); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn openai_stream_rejects_missing_capability_before_execution() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); let error = invoke_openai_stream( &plan, &runtime, RunRequest::text("stream"), OpenAiStreamTransport { port: 1, token: "unused".to_string(), }, ) .expect_err("streaming capability must be required"); assert!(matches!( error, FabricError::UnsupportedRuntimeCapability { capability: "streaming", .. } )); let ordinary = invoke_runtime(&plan, &runtime, RunRequest::text("ordinary")) .expect("unsupported stream must not execute or poison runtime"); assert_eq!(ordinary.output["invocation_count"], 1); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn openai_stream_rejects_stale_plan_capability_before_execution() { let (root, mut plan) = local_host_plan("success"); plan.capabilities.streaming = true; let runtime = start_runtime(&plan).expect("start local host"); let error = invoke_openai_stream( &plan, &runtime, RunRequest::text("stream"), OpenAiStreamTransport { port: 1, token: "unused".to_string(), }, ) .expect_err("the resolved descriptor must also claim streaming"); assert!(matches!( error, FabricError::UnsupportedRuntimeCapability { capability: "streaming", .. } )); let ordinary = invoke_runtime(&plan, &runtime, RunRequest::text("ordinary")) .expect("unsupported stream must not execute or poison runtime"); assert_eq!(ordinary.output["invocation_count"], 1); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn openai_stream_debug_output_redacts_the_bearer_token() { let transport = OpenAiStreamTransport { port: 1234, token: "stream-secret".to_string(), }; let sink = OpenAiStreamSink { protocol_version: OpenAiStreamProtocolVersion::V1Alpha1, profile: OpenAiStreamProfile::ChatCompletionsChunkV1, host: OpenAiStreamHost::Ipv4Loopback, port: transport.port, token: transport.token.clone(), runtime_id: "runtime-1".to_string(), invocation_id: "invocation-1".to_string(), request_id: "request-1".to_string(), }; assert!(!format!("{transport:?}").contains("stream-secret")); assert!(!format!("{sink:?}").contains("stream-secret")); } #[test] fn openai_stream_transport_rejects_blank_or_header_unsafe_tokens() { for token in ["", " ", "safe\r\ninjected"] { let error = validate_openai_stream_transport(&OpenAiStreamTransport { port: 1234, token: token.to_string(), }) .expect_err("invalid token must be rejected"); assert!(matches!( error, FabricError::InvalidOpenAiStreamTransport { field: "token", .. } )); } } #[test] fn process_adapter_uses_the_same_persistent_local_host_protocol() { let (root, mut plan) = local_host_plan("success"); let descriptor = plan .adapter_descriptor .as_mut() .expect("resolved adapter descriptor"); descriptor.descriptor.adapter_kind = AdapterKind::Process; descriptor.descriptor.runner = serde_json::from_value(serde_json::json!({ "command": "python3", "script": "../../fake_host.py", })) .expect("process runner settings"); let runtime = start_runtime(&plan).expect("start process host"); let first = invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); let second = invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); stop_runtime(&plan, &runtime).expect("stop process host"); assert_eq!(first.output["host_pid"], second.output["host_pid"]); assert_eq!(first.output["invocation_count"], 1); assert_eq!(second.output["invocation_count"], 2); let _ = fs::remove_dir_all(root); } fn assert_unavailable(result: &Result, runtime_id: &str) { match result { Err(FabricError::ResourceObservationUnavailable { runtime_id: actual, .. }) => assert_eq!(actual, runtime_id), other => panic!("expected ResourceObservationUnavailable, got {other:?}"), } } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn runtime_resource_snapshot_of_active_runtime_reads_host_tree() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("snapshot"); stop_runtime(&plan, &runtime).expect("stop local host"); assert_eq!(snapshot.runtime_id, runtime.runtime_id); assert!(snapshot.process_count >= 1); assert!(snapshot.resident_memory_bytes > 0); assert!(snapshot.cpu_time_seconds >= 0.0); assert!(snapshot.started_at.ends_with('Z')); assert!(snapshot.started_at <= snapshot.observed_at); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn runtime_resource_snapshot_during_invocation_returns_before_invocation_ends() { let (root, plan) = local_host_plan("busy"); let runtime = start_runtime(&plan).expect("start local host"); let saw_descendant = thread::scope(|scope| { let invocation = scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("work"))); let mut saw_descendant = false; while !saw_descendant && !invocation.is_finished() { let snapshot = runtime_resource_snapshot(&plan, &runtime).expect("live snapshot"); saw_descendant = snapshot.process_count >= 2 && !invocation.is_finished(); thread::sleep(Duration::from_millis(20)); } invocation .join() .expect("invocation thread") .expect("invoke local host"); saw_descendant }); stop_runtime(&plan, &runtime).expect("stop local host"); assert!(saw_descendant, "no snapshot saw the running descendant"); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn invocation_result_carries_final_resource_snapshot_with_waited_descendant_cpu() { let (root, plan) = local_host_plan("busy"); let runtime = start_runtime(&plan).expect("start local host"); let result = invoke_runtime(&plan, &runtime, RunRequest::text("work")).expect("invoke local host"); stop_runtime(&plan, &runtime).expect("stop local host"); let snapshot = result.resource_snapshot.expect("final resource snapshot"); assert_eq!(snapshot.runtime_id, result.runtime_id); assert!(snapshot.process_count >= 1); assert!( snapshot.cpu_time_seconds >= 1.0, "descendant CPU missing: {snapshot:?}" ); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn invocation_stats_isolate_concurrent_work_on_a_warm_runtime() { let (root, plan) = local_host_plan("busy"); let runtime = start_runtime(&plan).expect("start local host"); let wall_started = Instant::now(); let mut results = thread::scope(|scope| { let first = scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("one"))); let second = scope.spawn(|| invoke_runtime(&plan, &runtime, RunRequest::text("two"))); vec![ first.join().unwrap().unwrap(), second.join().unwrap().unwrap(), ] }); let wall_elapsed = wall_started.elapsed().as_secs_f64(); stop_runtime(&plan, &runtime).expect("stop local host"); results.sort_by_key(|result| result.output["invocation_count"].as_u64().unwrap()); let first = results[0].system_stats.as_ref().expect("first stats"); let second = results[1].system_stats.as_ref().expect("second stats"); for result in &results { let stats = result.system_stats.as_ref().unwrap(); assert!(stats.elapsed_seconds > 0.0); let cpu = stats.cpu_time_seconds_estimate.expect("CPU estimate"); assert!(cpu >= 1.0, "waited child CPU missing: {stats:?}"); assert_eq!(stats.after.snapshot(), result.resource_snapshot.as_ref()); } assert!( second.before.snapshot().unwrap().cpu_time_seconds >= first.after.snapshot().unwrap().cpu_time_seconds ); assert!( second.cpu_time_seconds_estimate.unwrap() < second.after.snapshot().unwrap().cpu_time_seconds - 0.9, "second estimate includes previous invocation's CPU" ); assert!( first.elapsed_seconds + second.elapsed_seconds <= wall_elapsed, "elapsed measurements include time queued behind another invocation" ); assert_ne!(results[0].invocation_id, results[1].invocation_id); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn disabling_invocation_stats_survives_serialization_and_preserves_final_snapshot() { let (root, mut plan) = local_host_plan("echo"); assert!(plan.config.system_stats.is_some()); plan.config.system_stats = None; let mapping = serde_json::to_value(&plan.config).unwrap(); assert!(mapping.get("system_stats").unwrap().is_null()); let decoded: crate::FabricConfig = serde_json::from_value(mapping.clone()).unwrap(); assert!(decoded.system_stats.is_none()); let mut omitted = mapping; omitted.as_object_mut().unwrap().remove("system_stats"); let decoded: crate::FabricConfig = serde_json::from_value(omitted).unwrap(); assert_eq!(decoded.system_stats, Some(Default::default())); let result = run_plan(&plan, RunRequest::text("disabled")).unwrap(); assert!(result.system_stats.is_none()); assert!(result.resource_snapshot.is_some()); assert!( !result .artifacts .artifacts .iter() .any(|a| a.kind.starts_with("system_stats")) ); let home = Path::new(result.metadata["fabric_home"].as_str().unwrap()); assert!(!home.join("system-stats.jsonl").exists()); assert!(!home.join("system-stats-summary.json").exists()); let _ = fs::remove_dir_all(root); } #[test] fn filtered_sampling_does_not_collect_unselected_process_or_memory_groups() { let (root, mut plan) = local_host_plan("echo"); plan.config.system_stats = Some(crate::SystemStatsConfig { collect: vec![crate::SystemStatsMetricGroup::CgroupCpu], sampling: Some(Default::default()), }); let result = run_plan(&plan, RunRequest::text("filtered")).unwrap(); let stats = result.system_stats.unwrap(); assert!(matches!(stats.before, ResourceObservation::NotCollected)); assert!(matches!(stats.after, ResourceObservation::NotCollected)); assert!(stats.cpu_time_seconds_estimate.is_none()); assert!(matches!( stats.cgroup.unwrap().oom_kills, crate::Measurement::NotCollected )); let sampled = stats.sampled.unwrap(); assert!(sampled.resident_memory_bytes.is_none()); assert!(sampled.process_count.is_none()); assert!(matches!( sampled.cpu_time_seconds_estimate, crate::Measurement::NotCollected )); let rows: Vec = fs::read_to_string(sampled.samples.unwrap().path) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(rows.len(), 2); assert!( rows.iter() .all(|row| row["adapter_tree"]["status"] == "not_collected") ); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn sampled_stats_capture_transient_worker_memory_and_preserve_correlation() { let (root, mut plan) = local_host_plan("memory_spike"); plan.config.system_stats = Some(crate::SystemStatsConfig { sampling: Some(crate::SystemStatsSamplingConfig { interval_ms: 100, max_samples: 100, }), ..Default::default() }); let result = run_plan(&plan, RunRequest::text("spike")).expect("run with sampling"); let stats = result.system_stats.as_ref().unwrap(); let summary = stats.sampled.as_ref().unwrap(); let gauge = summary.resident_memory_bytes.as_ref().unwrap(); assert!( gauge.sampled_peak.unwrap() > stats.after.snapshot().unwrap().resident_memory_bytes + 128 * 1024 * 1024, "peak was lost after worker exit: {summary:?}" ); assert!(summary.sample_count >= 3); // A spike sample need not have a neighbor within twice the interval. assert!((0.0..=1.0).contains(&gauge.covered_fraction)); let path = &summary.samples.as_ref().unwrap().path; let rows: Vec = fs::read_to_string(path) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(rows.len(), summary.sample_count as usize); for row in &rows { assert_eq!(row["runtime_id"], result.runtime_id); assert_eq!(row["invocation_id"], result.invocation_id); assert_eq!(row["request_id"], result.request_id); assert!(row["offset_seconds"].as_f64().unwrap() <= stats.elapsed_seconds); } assert_eq!( rows.last().unwrap()["offset_seconds"].as_f64(), Some(stats.elapsed_seconds) ); let reference = result .artifacts .artifacts .iter() .find(|a| a.kind == "system_stats_summary") .unwrap(); let saved: Value = serde_json::from_str(&fs::read_to_string(&reference.path).unwrap()).unwrap(); assert_eq!(saved["invocation_id"], result.invocation_id); assert_eq!(saved["system_stats"], serde_json::to_value(stats).unwrap()); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn sampled_stats_keep_final_endpoint_when_retention_is_full() { let (root, mut plan) = local_host_plan("busy"); plan.config.system_stats = Some(crate::SystemStatsConfig { sampling: Some(crate::SystemStatsSamplingConfig { interval_ms: 100, max_samples: 2, }), ..Default::default() }); let result = run_plan(&plan, RunRequest::text("bounded")).unwrap(); let stats = result.system_stats.unwrap(); let before = stats.before.snapshot().unwrap(); let after = stats.after.snapshot().unwrap(); assert_eq!( stats.cpu_time_seconds_estimate, Some(after.cpu_time_seconds - before.cpu_time_seconds) ); if let Some(cgroup) = &stats.cgroup && let Some((before, after)) = cgroup.before.snapshot().zip(cgroup.after.snapshot()) && let Some((before, after)) = before .cpu_usage_microseconds .value() .zip(after.cpu_usage_microseconds.value()) { assert_eq!( cgroup.cpu_usage_microseconds, crate::Measurement::Available { value: after - before } ); } let summary = stats.sampled.unwrap(); assert!(matches!( summary.cpu_time_seconds_estimate, crate::Measurement::Unavailable { .. } )); assert_eq!(summary.sample_count, 2); assert!(summary.dropped_samples > 0); assert!(summary.truncated); let rows: Vec = fs::read_to_string(summary.samples.unwrap().path) .unwrap() .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); assert_eq!(rows.len(), 2); assert_eq!( rows[1]["offset_seconds"].as_f64(), Some(stats.elapsed_seconds) ); let _ = fs::remove_dir_all(root); } #[test] fn protocol_failure_retains_sample_evidence_without_replacing_the_error() { let (root, mut plan) = local_host_plan("invoke_failure"); plan.config.system_stats = Some(crate::SystemStatsConfig { sampling: Some(Default::default()), ..Default::default() }); let runtime = start_runtime(&plan).unwrap(); let error = invoke_runtime(&plan, &runtime, RunRequest::text("fail")).unwrap_err(); assert!( matches!(error, FabricError::AdapterLifecycleOperation { code, .. } if code == "fake_invoke") ); stop_runtime(&plan, &runtime).unwrap(); let path = root.join("artifacts/.fabric").join(runtime.runtime_id); let summaries: Vec<_> = fs::read_dir(path) .unwrap() .filter_map(|entry| { let path = entry.unwrap().path().join("system-stats-summary.json"); path.is_file().then_some(path) }) .collect(); assert_eq!(summaries.len(), 1); let summary: Value = serde_json::from_str(&fs::read_to_string(&summaries[0]).unwrap()).unwrap(); assert_eq!(summary["system_stats"]["sampled"]["sample_count"], 2); let _ = fs::remove_dir_all(root); } #[test] fn observation_failure_preserves_invocation_result() { let (root, plan) = local_host_plan("adapter_reported_failure"); let runtime = start_runtime(&plan).expect("start local host"); // Make observation unavailable without breaking the live adapter protocol. local_host_processes().remove(&runtime.runtime_id); let result = invoke_runtime(&plan, &runtime, RunRequest::text("fail")) .expect("normalized adapter failure still returns a result"); stop_runtime(&plan, &runtime).expect("stop local host"); assert_eq!(result.status, RunStatus::Failed); assert_eq!(result.error.unwrap().code, "fake_adapter_failure"); assert!(result.resource_snapshot.is_none()); let stats = result.system_stats.expect("duration and availability"); assert!(stats.elapsed_seconds > 0.0); assert!(stats.cpu_time_seconds_estimate.is_none()); for observation in [&stats.before, &stats.after] { #[cfg(any(target_os = "linux", target_os = "macos"))] assert!( matches!(observation, ResourceObservation::Unavailable { reason } if !reason.is_empty()) ); #[cfg(not(any(target_os = "linux", target_os = "macos")))] assert!( matches!(observation, ResourceObservation::Unsupported { reason } if !reason.is_empty()) ); } let _ = fs::remove_dir_all(root); } #[test] fn runtime_resource_snapshot_after_stop_is_unavailable() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); stop_runtime(&plan, &runtime).expect("stop local host"); let result = runtime_resource_snapshot(&plan, &runtime); if cfg!(any(target_os = "linux", target_os = "macos")) { assert_unavailable(&result, &runtime.runtime_id); } else { assert!(matches!( result, Err(FabricError::ResourceObservationUnsupported { .. }) )); } let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn runtime_resource_snapshot_after_host_exit_is_unavailable() { let (root, plan) = local_host_plan("crash_after_start"); let runtime = start_runtime(&plan).expect("start local host"); let deadline = Instant::now() + Duration::from_secs(10); let mut result = runtime_resource_snapshot(&plan, &runtime); while result.is_ok() && Instant::now() < deadline { thread::sleep(Duration::from_millis(50)); result = runtime_resource_snapshot(&plan, &runtime); } let _ = stop_runtime(&plan, &runtime); assert_unavailable(&result, &runtime.runtime_id); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn runtime_resource_snapshot_during_concurrent_stop_is_snapshot_or_unavailable() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); let results = thread::scope(|scope| { let snapshots = scope.spawn(|| { let mut results = Vec::new(); loop { let result = runtime_resource_snapshot(&plan, &runtime); let stopped = result.is_err(); results.push(result); if stopped { return results; } } }); thread::sleep(Duration::from_millis(50)); stop_runtime(&plan, &runtime).expect("stop local host"); snapshots.join().expect("snapshot thread") }); let (last, earlier) = results.split_last().expect("at least one snapshot"); assert!(earlier.iter().all(Result::is_ok)); assert_unavailable(last, &runtime.runtime_id); let _ = fs::remove_dir_all(root); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn runtime_resource_snapshot_after_invocation_timeout_is_unavailable() { let (root, plan) = local_host_plan("invoke_timeout"); let runtime = start_runtime(&plan).expect("start local host"); runtime_resource_snapshot(&plan, &runtime).expect("snapshot before the timeout"); run_local_host_adapter_with_timeout( &plan, &runtime, RunRequest::text("hang"), Duration::from_millis(100), ) .expect_err("unresponsive host must time out"); assert_unavailable( &runtime_resource_snapshot(&plan, &runtime), &runtime.runtime_id, ); let _ = stop_runtime(&plan, &runtime); let _ = fs::remove_dir_all(root); } #[test] fn runtime_resource_snapshot_of_non_local_environment_is_unsupported() { let (root, plan) = local_host_plan("success"); let mut environment = prepare_environment(&plan).expect("prepare environment"); environment.provider = "remote".to_string(); let runtime_id = new_id("runtime"); let runtime = RuntimeHandle { runtime_binding: runtime_binding(&runtime_id, None, &plan, &environment) .expect("runtime binding"), runtime_id, agent_name: plan.agent_name.clone(), harness: harness(&plan), adapter_kind: adapter_kind(&plan), adapter_id: adapter_id(&plan), service_id: None, environment, }; let result = runtime_resource_snapshot(&plan, &runtime); assert!( matches!( &result, Err(FabricError::ResourceObservationUnsupported { runtime_id, reason }) if *runtime_id == runtime.runtime_id && reason.contains("remote") ), "{result:?}" ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_relay_config_is_runtime_scoped() { let (root, plan) = local_host_plan_with_relay("success", true); let runtime = start_runtime(&plan).expect("start local host"); let host = local_hosts() .get(&runtime.runtime_id) .cloned() .expect("active local host"); let relay_config_path = host .lock() .expect("local host") .relay_config .as_ref() .expect("Relay config") .path .clone(); let relay_config: Value = serde_json::from_str( &fs::read_to_string(relay_config_path).expect("read Relay config"), ) .expect("parse Relay config"); assert_eq!( relay_config["fabric"]["runtime_id"], serde_json::json!(runtime.runtime_id) ); assert!(relay_config["fabric"].get("invocation_id").is_none()); assert!(relay_config["fabric"].get("request_id").is_none()); let result = invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("invoke local host"); assert_eq!(result.output["runtime_id"], result.runtime_id); assert_eq!(result.output["invocation_id"], result.invocation_id); assert_eq!(result.output["request_id"], result.request_id); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_start_failure_preserves_stage_and_diagnostics() { let (root, plan) = local_host_plan("start_failure"); let error = start_runtime(&plan).expect_err("start must fail"); let message = error.to_string(); assert!(message.contains("lifecycle start"), "{message}"); assert!(message.contains("fake_start"), "{message}"); assert!(message.contains("start diagnostic"), "{message}"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_supplies_fresh_adapter_deadlines() { let (root, mut plan) = local_host_plan("success"); plan.config.runtime.timeout_seconds = Some(60.0); let runtime = start_runtime(&plan).expect("start local host"); for input in ["first", "second"] { let before = now_millis(); let result = invoke_runtime(&plan, &runtime, RunRequest::text(input)).expect("invoke"); let deadline = u128::from(result.output["deadline_millis"].as_u64().unwrap()); // Adapters must return before Fabric's own 60-second timeout fires. assert!((before + 50_000..=now_millis() + 50_000).contains(&deadline)); } stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn short_invocation_timeouts_split_adapter_deadline_and_reserve() { let before = now_millis(); let deadline = u128::from(adapter_deadline_millis(Duration::from_secs(4)).unwrap()); // Half of a 4-second timeout remains for the adapter to clean up and return. assert!((before + 2_000..=now_millis() + 2_000).contains(&deadline)); } #[test] fn local_host_invoke_timeout_evicts_and_terminates_host() { let (root, plan) = local_host_plan("invoke_timeout"); let runtime = start_runtime(&plan).expect("start local host"); crate::check::register(&runtime, vec![crate::check::CheckLevel::Live]); let host = local_hosts() .get(&runtime.runtime_id) .cloned() .expect("active local host"); let runtime_dir = host.lock().expect("local host").runtime_dir.clone(); let error = run_local_host_adapter_with_timeout( &plan, &runtime, RunRequest::text("hang"), Duration::from_millis(100), ) .expect_err("unresponsive host must time out"); assert!(matches!( &error, FabricError::AdapterLifecycleOperation { code, diagnostics, .. } if code == "host_timeout" && diagnostics.contains("invoke accepted without response") )); assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!( host.lock() .expect("local host") .child .try_wait() .expect("inspect timed-out host") .is_some(), "timed-out host process must be terminated" ); assert!(!runtime_dir.exists()); #[cfg(any(target_os = "linux", target_os = "macos", windows))] assert_eq!(local_host_has_exited(&runtime.runtime_id), Some(true)); #[cfg(any(target_os = "linux", target_os = "macos"))] for level in ["live", "active", "ready"] { let request = serde_json::from_value(serde_json::json!({ "scope": "runtime", "runtime_id": runtime.runtime_id, "level": level, "timeout_seconds": 0.5, })) .unwrap(); let report = crate::check::check_environment(&runtime.environment, &request).unwrap(); assert_eq!(report.outcome, crate::check::CheckOutcome::Failed); assert_eq!( report.reason_code, Some(crate::check::CheckReason::AgentExited) ); assert_eq!( report.runtime.unwrap().state, crate::check::CheckRuntimeState::Failed ); } let retry = invoke_runtime(&plan, &runtime, RunRequest::text("retry")) .expect_err("timed-out runtime must remain unavailable"); assert!(matches!( retry, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_unavailable" )); let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); assert_eq!(stopped[0].metadata["already_stopped"], true); assert!(local_host_has_exited(&runtime.runtime_id).is_none()); let _ = fs::remove_dir_all(root); } #[cfg(unix)] #[test] fn local_host_invoke_timeout_kills_children_of_host_exiting_during_grace() { let (root, plan) = local_host_plan("invoke_timeout_exit"); let runtime = start_runtime(&plan).expect("start local host"); let child = fs::read_to_string(root.join("child.pid")) .expect("read host child pid") .parse::() .expect("host child pid"); let error = run_local_host_adapter_with_timeout( &plan, &runtime, RunRequest::text("hang"), Duration::from_millis(100), ) .expect_err("unresponsive host must time out"); assert!( matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" ), "{error}" ); // Orphans can linger as zombies until reaped; they no longer run. let running = || { let output = Command::new("ps") .args(["-o", "stat=", "-p", &child.to_string()]) .output() .expect("inspect host child"); let state = String::from_utf8_lossy(&output.stdout); let state = state.trim(); !state.is_empty() && !state.starts_with('Z') }; let deadline = Instant::now() + Duration::from_secs(5); while running() && Instant::now() < deadline { thread::sleep(Duration::from_millis(10)); } let survived = running(); if survived { // SAFETY: kill has no memory-safety preconditions. unsafe { libc::kill(child, libc::SIGKILL) }; } assert!(!survived, "host child survived the host's exit"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_rejects_zero_timeout_from_deserialized_plan() { let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).expect("start local host"); let mut invalid_plan = plan.clone(); invalid_plan.config.runtime.timeout_seconds = Some(0.0); let error = run_local_host_adapter(&invalid_plan, &runtime, RunRequest::text("first")) .expect_err("zero timeout must be rejected"); assert!(matches!( error, FabricError::InvalidConfig { field, .. } if field == "runtime.timeout_seconds" )); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_revalidates_harness_settings_before_runtime_start() { let (root, plan) = local_host_plan("success"); let mut invalid_plan = plan.clone(); invalid_plan .config .harness .as_mut() .expect("harness") .settings .insert("unknown".to_string(), Value::Bool(true)); let error = start_runtime(&invalid_plan).expect_err("start must reject invalid harness settings"); assert!(matches!( error, FabricError::InvalidHarnessSettings { adapter_id, settings_path, .. } if adapter_id == "acme.fabric.local-host" && settings_path == "harness.settings.unknown" )); let _ = fs::remove_dir_all(root); } #[test] fn local_host_revalidates_southbound_config_before_runtime_start() { for field in ["provider", "model"] { let (root, plan) = local_host_plan("success"); let mut serialized = serde_json::to_value(plan).expect("serialize plan"); serialized["agent_config"]["models"]["primary"] = serde_json::json!({ "provider": "nvidia", "model": "test-model" }); serialized["agent_config"]["models"]["primary"][field] = Value::String(" \t".to_string()); let plan: RunPlan = serde_json::from_value(serialized).expect("deserialize run plan"); let error = start_runtime(&plan).expect_err("start must reject blank agent config"); assert!(matches!( error, FabricError::InvalidConfig { field: actual, .. } if actual == format!("agent_config.models.primary.{field}") )); assert!(!root.join("artifacts").exists()); let _ = fs::remove_dir_all(root); } } #[test] fn local_host_revalidates_workflow_before_runtime_start() { let (root, mut plan) = local_host_plan("success"); plan.config.workflow = Some( serde_json::from_value(serde_json::json!({ "target_id": "test.fabric.workflow", "settings": {"llm_name": 7} })) .expect("typed workflow"), ); let descriptor: crate::config::AdapterTargetDescriptor = serde_json::from_value(serde_json::json!({ "contract_version": crate::ADAPTER_CONTRACT_VERSION, "type": "workflow", "id": "test.fabric.workflow", "adapter_id": "acme.fabric.local-host", "spec": { "entrypoint": { "kind": "workflow_registry", "ref": "test_agent" }, "settings_schema": { "type": "object", "properties": {"llm_name": {"type": "string"}} } } })) .expect("target descriptor"); plan.adapter_target_descriptor = Some(crate::config::ResolvedAdapterTargetDescriptor { provenance: vec![crate::config::DescriptorProvenance { source: crate::config::DescriptorSource::ExplicitLocal, path: root.join("workflow.fabric-target.json"), root: root.clone(), }], descriptor, }); let error = start_runtime(&plan).expect_err("start must reject invalid workflow"); assert!(matches!( error, FabricError::InvalidWorkflow { workflow_path, .. } if workflow_path == "workflow.settings.llm_name" )); let _ = fs::remove_dir_all(root); } #[test] fn local_host_revalidates_projected_workflow_entrypoint_before_runtime_start() { for (field, value) in [ ("agent_config.workflow.entrypoint.kind", " "), ("agent_config.workflow.entrypoint.ref", "\t"), ] { let (root, plan) = local_host_plan("start_failure"); let mut serialized = serde_json::to_value(plan).expect("serialize plan"); serialized["agent_config"]["workflow"] = serde_json::json!({ "entrypoint": { "kind": "workflow_registry", "ref": "test_agent" } }); serialized["agent_config"]["workflow"]["entrypoint"][field .strip_prefix("agent_config.workflow.entrypoint.") .expect("entrypoint field")] = Value::String(value.to_string()); let plan: RunPlan = serde_json::from_value(serialized).expect("deserialize plan"); let error = start_runtime(&plan).expect_err("start must reject blank entrypoint"); assert!(matches!( error, FabricError::InvalidConfig { field: actual, .. } if actual == field )); assert!(!root.join("artifacts").exists()); let _ = fs::remove_dir_all(root); } } #[test] fn local_host_revalidates_scalar_compatibility_at_runtime_boundaries() { let (root, plan) = local_host_plan("success"); let mut incompatible_plan = plan.clone(); incompatible_plan.config.runtime.max_turns = Some(1); let start_error = start_runtime(&incompatible_plan).expect_err("start must reject incompatibility"); assert!(matches!( start_error, FabricError::AdapterCompatibility { field, .. } if field == "runtime.max_turns" )); let runtime = start_runtime(&plan).expect("start local host"); let invoke_error = invoke_runtime(&incompatible_plan, &runtime, RunRequest::text("first")) .expect_err("invoke must reject incompatibility"); assert!(matches!( invoke_error, FabricError::AdapterCompatibility { field, .. } if field == "runtime.max_turns" )); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_timeout_prevents_waiting_invocation_from_reusing_host() { let (root, plan) = local_host_plan("invoke_timeout"); let runtime = start_runtime(&plan).expect("start local host"); let host = local_hosts() .get(&runtime.runtime_id) .cloned() .expect("active local host"); let stderr_path = host.lock().expect("local host").stderr_path.clone(); let first_plan = plan.clone(); let first_runtime = runtime.clone(); let first = thread::spawn(move || { run_local_host_adapter_with_timeout( &first_plan, &first_runtime, RunRequest::text("first"), Duration::from_millis(500), ) }); let accepted_deadline = Instant::now() + Duration::from_secs(2); while !fs::read_to_string(&stderr_path) .expect("read host stderr") .contains("invoke accepted without response") { assert!( Instant::now() < accepted_deadline, "first invocation was not accepted" ); thread::sleep(Duration::from_millis(10)); } let second_plan = plan.clone(); let second_runtime = runtime.clone(); let second = thread::spawn(move || { run_local_host_adapter_with_timeout( &second_plan, &second_runtime, RunRequest::text("second"), Duration::from_millis(100), ) }); // The registry, this test, and the first invocation own three host // references. A fourth proves that the second invocation passed the // registry lookup and is waiting for the host mutex. let waiting_deadline = Instant::now() + Duration::from_secs(1); while Arc::strong_count(&host) < 4 { assert!( Instant::now() < waiting_deadline, "second invocation did not acquire the host reference" ); thread::sleep(Duration::from_millis(10)); } let first_error = first .join() .expect("first invocation thread") .expect_err("first invocation must time out"); let second_error = second .join() .expect("second invocation thread") .expect_err("waiting invocation must not reuse the timed-out host"); assert!(matches!( first_error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" )); assert!(matches!( second_error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_crashed" )); assert!(!local_hosts().contains_key(&runtime.runtime_id)); let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); assert_eq!(stopped[0].metadata["already_stopped"], true); let _ = fs::remove_dir_all(root); } #[test] fn local_host_captures_each_stderr_delta_once() { let (root, plan) = local_host_plan("invoke_stderr"); let runtime = start_runtime(&plan).expect("start local host"); let first = invoke_runtime(&plan, &runtime, RunRequest::text("first")).expect("first invocation"); let second = invoke_runtime(&plan, &runtime, RunRequest::text("second")).expect("second invocation"); let read_stderr = |result: &RunResult| { let artifact = result .artifacts .artifacts .iter() .find(|artifact| artifact.name == "stderr") .expect("stderr artifact"); fs::read_to_string(&artifact.path).expect("read stderr artifact") }; assert_eq!(read_stderr(&first), "diagnostic-1\n"); assert_eq!(read_stderr(&second), "diagnostic-2\n"); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn run_plan_stops_local_host_after_invoke_failure() { let (root, mut plan) = local_host_plan("invoke_failure"); let agent_name = new_id("local-host-invoke-error-agent"); plan.agent_name = agent_name.clone(); plan.config.metadata.name = agent_name.clone(); let error = run_plan(&plan, RunRequest::text("fail")).expect_err("invoke must fail"); assert!(error.to_string().contains("fake_invoke"), "{error}"); assert!( stopped_agents().contains(&agent_name), "run_plan must stop the local host after invocation failure" ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_preserves_normalized_adapter_failure() { let (root, plan) = local_host_plan("adapter_reported_failure"); let result = run_plan(&plan, RunRequest::text("fail")).expect("normalized result"); assert_eq!(result.status, RunStatus::Failed); assert_eq!(result.output["input"], "fail"); assert_eq!( result.error, Some(ErrorInfo { stage: ErrorStage::Invoke, code: "fake_adapter_failure".to_string(), message: "adapter rejected the invocation".to_string(), retryable: true, metadata: BTreeMap::from([("source".to_string(), serde_json::json!("fake-host"))]), }) ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_rejects_blank_agent_result_fields() { let (root, plan) = local_host_plan("blank_error_code"); let error = run_plan(&plan, RunRequest::text("fail")) .expect_err("blank adapter result fields must fail runtime validation"); assert!( matches!( &error, FabricError::AdapterLifecycleOperation { code, message, .. } if code == "invalid_agent_run_result" && message.contains("error.code must be a non-blank string") ), "{error}" ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_enriches_typed_usage_artifacts_and_extensions() { let (root, plan) = local_host_plan("typed_result"); let result = run_plan(&plan, RunRequest::text("enrich")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); assert_eq!( result.usage, Some(RunUsage { input_tokens: Some(3), cached_input_tokens: Some(1), input_tokens_include_cache: Some(true), output_tokens: Some(5), total_tokens: Some(8), cost_usd: Some(0.25), metadata: BTreeMap::from([("provider".to_string(), serde_json::json!("fake"),)]), }) ); let artifact = result .artifacts .artifacts .iter() .find(|artifact| artifact.name == "report") .expect("adapter artifact"); assert!(artifact.path.ends_with("reports/result.json")); assert_eq!( artifact.metadata, BTreeMap::from([("source".to_string(), serde_json::json!("fake-host"))]) ); assert_eq!( result.metadata.get("adapter"), Some(&serde_json::json!({"trace_id": "trace-1"})) ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_preserves_detailed_usage_in_metadata() { let (root, plan) = local_host_plan("detailed_usage_result"); let result = run_plan(&plan, RunRequest::text("usage")).expect("normalized result"); assert_eq!(result.status, RunStatus::Succeeded); let usage = result.usage.expect("run usage"); assert_eq!( usage, RunUsage { input_tokens: Some(3), cached_input_tokens: Some(1), input_tokens_include_cache: Some(true), output_tokens: Some(5), total_tokens: Some(8), cost_usd: Some(0.25), metadata: BTreeMap::from([ ("provider".to_string(), serde_json::json!("fake")), ( "nemo_fabric.detailed_token_usage.v1".to_string(), serde_json::json!({ "input_tokens": 20, "cached_input_tokens": 5, "cache_write_input_tokens": 3, "input_tokens_include_cache": false, "output_tokens": 5, "reasoning_tokens": 4, "output_tokens_include_reasoning": false, "total_tokens": 37, "peak_request_input_tokens": 18, "models": [{ "model": "test-model", "provider": "fake", "input_tokens": 20, "reasoning_tokens": 4 }] }), ), ]), } ); let _ = fs::remove_dir_all(root); } #[test] fn local_host_rejects_typed_artifacts_without_runtime_root() { let (root, mut plan) = local_host_plan("typed_result"); plan.config.runtime.artifacts = None; plan.environment_plan .as_mut() .expect("resolved environment plan") .artifacts = None; let runtime = start_runtime(&plan).expect("start local host"); let error = invoke_runtime(&plan, &runtime, RunRequest::text("enrich")) .expect_err("adapter artifacts require a runtime artifact root"); assert!(matches!( &error, FabricError::AdapterLifecycleOperation { code, message, .. } if code == "invalid_agent_run_result" && message.contains("configured runtime artifact root") )); stop_runtime(&plan, &runtime).expect("stop local host"); let _ = fs::remove_dir_all(root); } #[test] fn relay_otlp_artifacts_take_their_media_type_from_the_sink_format() { let root = std::env::temp_dir().join(new_id("fabric-relay-otlp-test")); fs::create_dir_all(&root).expect("create root"); for name in ["trace.jsonl", "trace.otlp.pb", "trace.json", "trace.csv"] { fs::write(root.join(name), b"").expect("write artifact"); } let mut manifest = ArtifactManifest { root: Some(root.clone()), artifacts: Vec::new(), }; let output = serde_json::json!({"relay_artifacts": [ {"kind": "otlp", "path": "trace.jsonl", "format": "proto"}, {"kind": "otlp", "path": "trace.otlp.pb"}, {"kind": "otlp", "path": "trace.json", "format": "json_lines"}, {"kind": "otlp", "path": "trace.csv", "format": "csv"}, ]}); promote_relay_artifacts_to_manifest(&output, &mut manifest); let media_types: Vec<_> = manifest .artifacts .iter() .map(|artifact| (artifact.kind.as_str(), artifact.media_type.as_deref())) .collect(); assert_eq!( media_types, [ ("otlp", Some("application/x-protobuf")), ("otlp", Some("application/x-ndjson")), ("otlp", Some("application/x-ndjson")), ("otlp", None), ] ); let _ = fs::remove_dir_all(root); } #[test] fn promoted_agent_artifact_names_do_not_shadow_existing_artifacts() { let root = PathBuf::from("artifacts"); let mut manifest = ArtifactManifest { root: Some(root.clone()), artifacts: vec![ArtifactRef { name: "stdout".to_string(), kind: "log".to_string(), path: root.join("stdout.txt"), media_type: Some("text/plain".to_string()), metadata: BTreeMap::new(), }], }; let agent_artifacts = vec![AgentArtifact { name: "stdout".to_string(), kind: "report".to_string(), path: PathBuf::from("reports/result.json"), media_type: Some("application/json".to_string()), extensions: BTreeMap::new(), }]; promote_agent_artifacts_to_manifest(&agent_artifacts, &mut manifest); assert_eq!(manifest.artifacts.len(), 2); assert_eq!(manifest.artifacts[1].name, "stdout_2"); assert_eq!(manifest.artifacts[1].path, root.join("reports/result.json")); } #[test] fn agent_run_request_projection_keeps_only_southbound_fields() { let request = RunRequest { request_id: "request-1".to_string(), relay_session_root: None, input: serde_json::json!({"task": "review"}), context: BTreeMap::from([("rollout".to_string(), serde_json::json!(2))]), overrides: Some(serde_json::json!({"acme": {"mode": "strict"}})), }; let projected = project_agent_run_request(&request).expect("project request"); assert_eq!(projected.input, serde_json::json!({"task": "review"})); assert_eq!(projected.context, request.context); assert_eq!( projected.extensions, BTreeMap::from([("acme".to_string(), serde_json::json!({"mode": "strict"}))]) ); assert!( !serde_json::to_value(projected) .expect("serialize request") .as_object() .expect("request object") .contains_key("request_id") ); } #[test] fn typed_session_root_projects_into_adapter_request() { let root = "018f47a4-3af7-7d94-8e61-9f0f89b5d312"; let mut request = RunRequest::text("first turn"); request.relay_session_root = Some(root.to_string()); request.context.insert( "relay_session_root".to_string(), serde_json::json!("legacy"), ); let projected = project_agent_run_request(&request).expect("project request"); assert_eq!(projected.relay_session_root.as_deref(), Some(root)); assert_eq!(projected.context["relay_session_root"], "legacy"); let wire = serde_json::to_value(&projected).unwrap(); assert_eq!(wire["relay_session_root"], root); let decoded_adapter: AgentRunRequest = serde_json::from_value(wire).unwrap(); assert_eq!(decoded_adapter, projected); assert_eq!(request.context["relay_session_root"], "legacy"); let decoded: RunRequest = serde_json::from_value(serde_json::to_value(&request).unwrap()).unwrap(); assert_eq!(decoded.relay_session_root.as_deref(), Some(root)); request.relay_session_root = None; let projected = project_agent_run_request(&request).unwrap(); assert_eq!(projected.relay_session_root, None); assert_eq!(projected.context["relay_session_root"], "legacy"); assert!( serde_json::to_value(projected) .unwrap() .get("relay_session_root") .is_none() ); } #[test] fn agent_run_request_projection_rejects_non_object_overrides() { let mut request = RunRequest::text("review"); request.overrides = Some(serde_json::json!(["invalid"])); let error = project_agent_run_request(&request).expect_err("invalid overrides"); assert!(error.to_string().contains("request.overrides"), "{error}"); } #[test] #[cfg(any(target_os = "linux", target_os = "macos"))] fn health_reports_exited_hosts_for_all_health_adapters_without_the_invocation_lock() { use crate::check::{ CheckLevel, CheckOutcome, CheckReason, CheckRequest, CheckRuntimeState, }; let (root, plan) = local_host_plan("success"); let mut runtime = start_runtime(&plan).expect("start local host"); let original_adapter_id = runtime.adapter_id.clone(); let mut adapters = Vec::new(); let adapter_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../adapters"); for language in ["python", "typescript"] { for entry in fs::read_dir(adapter_root.join(language)).unwrap() { let directory = entry.unwrap().path(); if !directory.is_dir() || directory.file_name().unwrap() == "node_modules" { continue; } for file in fs::read_dir(directory).unwrap() { let file = file.unwrap().path(); if !file.to_string_lossy().ends_with(".fabric-adapter.json") { continue; } let descriptor: Value = serde_json::from_slice(&fs::read(file).unwrap()).unwrap(); if let Some(levels) = descriptor["capabilities"]["metadata"]["health_checks"] .as_array() .filter(|levels| !levels.is_empty()) { adapters.push(( descriptor["adapter_id"].as_str().unwrap().to_owned(), serde_json::from_value::>(Value::Array(levels.clone())) .unwrap(), )); } } } } assert!( !adapters.is_empty(), "health-capable adapter descriptors exist" ); // Third-party adapters must receive the same evidence without an ID allowlist. adapters.push(("acme.fabric.health-test".into(), vec![CheckLevel::Live])); let request = serde_json::from_value::(serde_json::json!({ "scope": "runtime", "runtime_id": runtime.runtime_id, "level": "ready", "timeout_seconds": 0.5, })) .unwrap(); let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); // An invocation owns this lock. Health must not acquire the lock. let mut host = host.lock().unwrap(); for (adapter_id, levels) in &adapters { runtime.adapter_id = Some(adapter_id.clone()); crate::check::register(&runtime, levels.clone()); let before = crate::check::check_environment(&runtime.environment, &request).unwrap(); assert_eq!(before.outcome, CheckOutcome::Unknown, "{adapter_id}"); assert_eq!( before.reason_code, Some(CheckReason::ProbeError), "{adapter_id}" ); } let process = local_host_processes() .get(&runtime.runtime_id) .unwrap() .clone(); host.child.kill().expect("kill the owned adapter host"); let deadline = Instant::now() + Duration::from_secs(2); while snapshot_local_host(&runtime.runtime_id, Some(&process)).is_ok() { assert!(Instant::now() < deadline, "killed host did not exit"); thread::sleep(Duration::from_millis(5)); } // Cover both an unreaped exited process and one whose PID is gone. for reaped in [false, true] { if reaped { host.child.wait().expect("reap exited host"); } for (adapter_id, levels) in &adapters { runtime.adapter_id = Some(adapter_id.clone()); crate::check::register(&runtime, levels.clone()); let report = crate::check::check_environment(&runtime.environment, &request).unwrap(); assert_eq!( report.outcome, CheckOutcome::Failed, "{adapter_id}: {report:?}" ); assert_eq!( report.reason_code, Some(CheckReason::AgentExited), "{adapter_id}" ); assert_eq!( report.runtime.unwrap().state, CheckRuntimeState::Failed, "{adapter_id}" ); assert_eq!( report.checks.len(), 1, "later levels must not run: {adapter_id}" ); assert_eq!(report.checks[0].name, crate::check::CheckStage::Live); assert_eq!( report.checks[0].source, crate::check::CheckSource::AdapterLiveness ); assert!(report.checks[0].observed_at.is_some()); assert!(report.elapsed_ms < 500.0); } } // Without retained process identity, a failed connection remains inconclusive. local_host_processes().remove(&runtime.runtime_id); crate::check::register(&runtime, vec![CheckLevel::Live]); let missing = crate::check::check_environment(&runtime.environment, &request).unwrap(); assert_eq!(missing.outcome, CheckOutcome::Unknown); assert_eq!(missing.reason_code, Some(CheckReason::ProbeError)); local_host_processes().insert(runtime.runtime_id.clone(), process); crate::check::register(&runtime, vec![]); let unsupported = crate::check::check_environment(&runtime.environment, &request).unwrap(); assert_eq!(unsupported.outcome, CheckOutcome::Unsupported); assert_eq!( unsupported.reason_code, Some(CheckReason::UnsupportedAdapter) ); assert!(unsupported.checks.is_empty()); crate::check::invalidate_runtime(&runtime); runtime.adapter_id = original_adapter_id; drop(host); stop_runtime(&plan, &runtime).expect("clean up exited host"); let _ = fs::remove_dir_all(root); } #[test] #[cfg(any(target_os = "linux", target_os = "macos"))] fn local_host_exit_evidence_survives_registry_contention_and_poisoning() { use std::sync::mpsc; let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); { let mut host = host.lock().unwrap(); host.child.kill().unwrap(); host.child.wait().unwrap(); } let runtime_id = runtime.runtime_id.clone(); let (entered_tx, entered_rx) = mpsc::channel(); let (observed_tx, observed_rx) = mpsc::channel(); let processes = local_host_processes(); let observer = thread::spawn(move || { entered_tx.send(()).unwrap(); observed_tx .send(local_host_has_exited(&runtime_id)) .unwrap(); }); entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); let early = observed_rx.recv_timeout(Duration::from_millis(50)); drop(processes); let contended = match early { Ok(observation) => observation, Err(mpsc::RecvTimeoutError::Timeout) => { observed_rx.recv_timeout(Duration::from_secs(2)).unwrap() } Err(error) => panic!("receive exit observation: {error}"), }; observer.join().unwrap(); let poisoning = std::panic::catch_unwind(|| { let _processes = local_host_processes(); panic!("poison the process registry for this test"); }); let poisoned = local_host_has_exited(&runtime.runtime_id); LOCAL_HOST_PROCESSES.clear_poison(); stop_runtime(&plan, &runtime).unwrap(); fs::remove_dir_all(root).unwrap(); assert!(poisoning.is_err()); assert_eq!((contended, poisoned), (Some(true), Some(true))); } #[test] #[cfg(any(target_os = "linux", target_os = "macos"))] fn health_preserves_a_host_exit_during_a_failed_probe() { use crate::check::{CheckOutcome, CheckReason, CheckRequest}; use std::os::unix::fs::PermissionsExt; let (root, plan) = local_host_plan("success"); let runtime = start_runtime(&plan).unwrap(); crate::check::register(&runtime, vec![crate::check::CheckLevel::Live]); let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); let host = host.lock().unwrap(); let pid = host.child.id(); let process = local_host_processes() .get(&runtime.runtime_id) .unwrap() .clone(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let endpoint = local_host_directory(&runtime.runtime_id).join("health.endpoint.json"); fs::write( &endpoint, serde_json::to_vec(&serde_json::json!({ "protocol_version": "fabric.check/v1alpha1", "runtime_id": runtime.runtime_id, "port": listener.local_addr().unwrap().port(), "token": "0123456789abcdef".repeat(4), })) .unwrap(), ) .unwrap(); fs::set_permissions(&endpoint, fs::Permissions::from_mode(0o600)).unwrap(); let exited = thread::spawn(move || { listener.set_nonblocking(true).unwrap(); let accept_deadline = Instant::now() + Duration::from_secs(5); let (mut stream, _) = loop { match listener.accept() { Ok(connection) => break connection, Err(error) if error.kind() == ErrorKind::WouldBlock && Instant::now() < accept_deadline => { thread::sleep(Duration::from_millis(10)); } Err(error) if error.kind() == ErrorKind::WouldBlock => { panic!("timed out waiting for health connection") } Err(error) => panic!("accept health connection: {error}"), } }; stream.set_nonblocking(false).unwrap(); stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); let mut request = [0; 1024]; assert!(stream.read(&mut request).unwrap() > 0); // SAFETY: pid belongs to the still-owned host whose invocation lock is held. assert_eq!(unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) }, 0); let deadline = Instant::now() + Duration::from_secs(2); while process.has_exited() != Some(true) { assert!(Instant::now() < deadline, "owned host did not exit"); thread::sleep(Duration::from_millis(1)); } // Dropping the connection simulates a health channel lost with its host. }); let request = serde_json::from_value::(serde_json::json!({ "scope": "runtime", "runtime_id": runtime.runtime_id, "timeout_seconds": 3.0, })) .unwrap(); let report = crate::check::check_environment(&runtime.environment, &request).unwrap(); exited.join().unwrap(); drop(host); stop_runtime(&plan, &runtime).unwrap(); fs::remove_dir_all(root).unwrap(); assert_eq!(report.outcome, CheckOutcome::Failed, "{report:?}"); assert_eq!(report.reason_code, Some(CheckReason::AgentExited)); assert!(report.checks[0].observed_at.is_some()); } #[test] fn local_host_crash_is_terminal_for_runtime() { let (root, plan) = local_host_plan("crash_after_start"); let runtime = start_runtime(&plan).expect("start local host"); let host = local_hosts() .get(&runtime.runtime_id) .cloned() .expect("active local host"); let stderr_path = host.lock().expect("local host").stderr_path.clone(); let closed_deadline = Instant::now() + Duration::from_secs(2); while !fs::read_to_string(&stderr_path) .expect("read host stderr") .contains("host crashed intentionally") { assert!( Instant::now() < closed_deadline, "host did not close its lifecycle pipe" ); thread::sleep(Duration::from_millis(10)); } assert!( host.lock() .expect("local host") .child .try_wait() .expect("inspect crashed host") .is_none(), "host must still be alive while its lifecycle pipe is closed" ); let first = invoke_runtime(&plan, &runtime, RunRequest::text("first")) .expect_err("crashed host must reject invocation"); let second = invoke_runtime(&plan, &runtime, RunRequest::text("second")) .expect_err("dead runtime must remain unusable"); assert!(first.to_string().contains("host_crashed"), "{first}"); assert!( first.to_string().contains("failed to send invoke"), "{first}" ); assert!(second.to_string().contains("host_crashed"), "{second}"); let stopped = stop_runtime(&plan, &runtime).expect("crashed host cleanup"); assert_eq!(stopped[0].metadata["host_crashed"], true); stop_runtime(&plan, &runtime).expect("cleanup remains idempotent"); let _ = fs::remove_dir_all(root); } #[test] fn local_host_stop_failure_still_releases_host() { let (root, plan) = local_host_plan("stop_failure"); let runtime = start_runtime(&plan).expect("start local host"); let error = stop_runtime(&plan, &runtime).expect_err("stop must fail"); assert!(error.to_string().contains("fake_stop"), "{error}"); let retry = stop_runtime(&plan, &runtime).expect("cleanup retry is idempotent"); assert_eq!(retry[0].metadata["already_stopped"], true); let _ = fs::remove_dir_all(root); } #[test] fn run_plan_preserves_completed_result_when_stop_fails() { let (root, plan) = local_host_plan("stop_failure"); let result = run_plan(&plan, RunRequest::text("completed")) .expect("completed result with stop failure"); assert_eq!(result.status, RunStatus::Failed); assert_eq!(result.output["input"], "completed"); let error = result.error.expect("stop error"); assert_eq!(error.stage, ErrorStage::Stop); assert_eq!(error.code, "runtime_stop_failed"); assert!(error.message.contains("fake_stop"), "{}", error.message); let _ = fs::remove_dir_all(root); } #[test] fn default_environment_prepares_through_local_provider() { let (root, mut plan) = local_host_plan("success"); plan.environment_plan = None; plan.config.environment = None; let environment = prepare_environment(&plan).expect("prepare default environment"); assert_eq!(environment.provider, "local"); assert_eq!( environment.control_location, ControlLocation::ExternalControl ); assert_eq!(environment.ownership, EnvironmentOwnership::CallerOwned); assert_eq!(environment.workspace, Some(root.clone())); assert_eq!(environment.artifacts, Some(root.join("artifacts"))); assert!(environment.env.is_empty()); assert!(environment.connection.is_empty()); assert!(environment.metadata.is_empty()); release_environment(&environment).expect("release local environment"); let _ = fs::remove_dir_all(root); } #[test] fn explicit_local_provider_preserves_resolved_environment_fields() { let (root, mut plan) = local_host_plan("success"); let environment_plan = plan .environment_plan .as_mut() .expect("resolved environment plan"); environment_plan.control_location = ControlLocation::InEnvControl; environment_plan.ownership = EnvironmentOwnership::FabricOwned; environment_plan.workspace = Some(root.join("workspace")); environment_plan.artifacts = Some(root.join("collected")); environment_plan .connection .insert("endpoint".to_string(), serde_json::json!("local")); environment_plan .settings .insert("setting".to_string(), serde_json::json!(true)); environment_plan .settings .insert("shared".to_string(), serde_json::json!("setting")); environment_plan .metadata .insert("shared".to_string(), serde_json::json!("consumer")); let environment = prepare_environment(&plan).expect("prepare explicit local environment"); assert_eq!(environment.provider, "local"); assert_eq!(environment.control_location, ControlLocation::InEnvControl); assert_eq!(environment.ownership, EnvironmentOwnership::FabricOwned); assert_eq!(environment.workspace, Some(root.join("workspace"))); assert_eq!(environment.artifacts, Some(root.join("collected"))); assert_eq!( environment.env.get("FABRIC_NORMALIZED_ENV"), Some(&"visible".to_string()) ); assert_eq!( environment.connection.get("endpoint"), Some(&serde_json::json!("local")) ); assert_eq!( environment.metadata.get("setting"), Some(&serde_json::json!(true)) ); assert_eq!( environment.metadata.get("shared"), Some(&serde_json::json!("consumer")) ); let _ = fs::remove_dir_all(root); } #[test] fn environment_preparation_rejects_unregistered_provider() { let (root, mut plan) = local_host_plan("success"); plan.environment_plan .as_mut() .expect("resolved environment plan") .provider = "unregistered".to_string(); let error = prepare_environment(&plan).expect_err("provider must be registered"); assert!(matches!( error, FabricError::UnsupportedEnvironmentProvider { provider, adapter_kind: AdapterKind::Python, } if provider == "unregistered" )); let _ = fs::remove_dir_all(root); } #[test] fn adapter_service_evidence_preserves_infrastructure_references() { let mut adapter = crate::ExecutionEvidence::gap("adapter", "fixture"); adapter.append(crate::ExecutionEvidence::gap("service", "fixture")); adapter.software_environments[1].status = crate::provenance::InventoryStatus::Available; adapter.software_environments[1].complete = true; adapter.software_environments[1].reason = None; let observation = serde_json::json!({ "status": "available", "value": "fixture-engine", "source": "owner:existing_report", "origin": "reported", "observed_at": "2026-10-27T12:00:00Z" }); adapter.infrastructure = Some( serde_json::from_value(serde_json::json!({ "container": { "context_refs": [adapter.software_environments[1].context_ref], "engine": observation, "engine_version": observation } })) .unwrap(), ); assert!(adapter.valid()); let retained = service_evidence(&adapter).unwrap(); let mut report = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence(&mut report, &adapter, Some(&retained)); assert!(report.valid()); assert_eq!(report.software_environments.len(), 3); assert_eq!(report.software_environments[2].participants, ["service"]); assert_eq!( report .infrastructure .as_ref() .unwrap() .container .as_ref() .unwrap() .context_refs, [report.software_environments[2].context_ref.clone()] ); } fn retained_service_fixture() -> crate::ExecutionEvidence { serde_json::from_value(serde_json::json!({ "software_environments": [{ "context_ref": "startup-service", "participants": ["service"], "status": "available", "complete": true, "entries": [{"role": "harness", "kind": "component", "name": "server", "version": {"status": "available", "value": "1.0+startup", "source": "service:startup", "origin": "reported", "observed_at": "2026-10-27T12:00:00Z"}}] }], "infrastructure": {"container": { "context_refs": ["startup-service"], "engine": {"status": "available", "value": "docker", "source": "owner:startup", "origin": "reported", "observed_at": "2026-10-27T12:00:00Z"}, "engine_version": {"status": "available", "value": "28.0", "source": "owner:startup", "origin": "reported", "observed_at": "2026-10-27T12:00:00Z"} }} })).unwrap() } #[test] fn retained_service_infrastructure_survives_missing_current_report() { let retained = service_evidence(&retained_service_fixture()).unwrap(); let mut result = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence( &mut result, &crate::ExecutionEvidence::gap("adapter", "missing"), Some(&retained), ); assert!(result.valid()); assert_eq!( result.software_environments[2].entries, retained.software_environments[0].entries ); assert_eq!( result .infrastructure .unwrap() .container .unwrap() .context_refs, ["env-2"] ); } #[test] fn partial_service_reports_preserve_retained_startup_and_deduplicate_infrastructure() { for complete in [false, true] { let retained = retained_service_fixture(); let mut adapter = retained.clone(); adapter.software_environments[0].entries.clear(); adapter.software_environments[0].complete = complete; adapter.software_environments[0].reason = (!complete).then(|| "partial".into()); let mut result = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence(&mut result, &adapter, Some(&retained)); assert!(result.valid()); assert_eq!(result.software_environments.len(), 3); assert_eq!( result.software_environments[2].entries, retained.software_environments[0].entries ); assert_eq!( result .infrastructure .unwrap() .container .unwrap() .context_refs, ["env-1", "env-2"] ); } } #[test] fn unavailable_reconnect_cannot_replace_shared_startup_snapshot() { let retained = retained_service_fixture(); let current = crate::ExecutionEvidence::gap("service", "unavailable"); let saved = retain_service_snapshot(current, Some(&retained)); assert_eq!(saved, retained); assert_eq!(retain_service_snapshot(saved.clone(), Some(&saved)), saved); let mut next_connection = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence( &mut next_connection, &crate::ExecutionEvidence::gap("adapter", "missing"), Some(&saved), ); assert!(next_connection.valid()); assert_eq!( next_connection.software_environments[2].entries[0] .version .value .as_deref(), Some("1.0+startup") ); } #[test] fn current_available_service_version_replaces_its_startup_observation() { let retained = retained_service_fixture(); let mut current = retained.clone(); current.software_environments[0].entries[0].version.value = Some("2.0+current".into()); current.software_environments[0].entries[0] .version .observed_at = "2026-10-28T12:00:00Z".into(); let saved = retain_service_snapshot(current.clone(), Some(&retained)); assert_eq!(saved.software_environments.len(), 1); assert_eq!( saved.software_environments[0].entries, current.software_environments[0].entries ); let mut result = crate::ExecutionEvidence::default(); append_adapter_evidence(&mut result, ¤t, Some(&retained)); assert!(result.valid()); assert_eq!(result.software_environments.len(), 1); assert_eq!( result.software_environments[0].entries[0] .version .value .as_deref(), Some("2.0+current") ); } #[test] fn conflicting_service_infrastructure_stays_an_explicit_gap() { let retained = retained_service_fixture(); let mut adapter = retained.clone(); adapter .infrastructure .as_mut() .unwrap() .container .as_mut() .unwrap() .engine_version .value = Some("29.0".into()); let mut result = crate::ExecutionEvidence::default(); append_adapter_evidence(&mut result, &adapter, Some(&retained)); assert!(result.valid()); assert_eq!( result.software_environments[0].reason.as_deref(), Some("conflicting_infrastructure_reports") ); } #[test] fn adapter_without_service_evidence_uses_the_retained_snapshot() { let adapter = crate::ExecutionEvidence::gap("adapter", "fixture"); let retained = crate::ExecutionEvidence::gap("service", "fixture"); let mut report = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence(&mut report, &adapter, Some(&retained)); assert!(report.valid()); assert_eq!(report.software_environments.len(), 3); assert_eq!(report.software_environments[2].participants, ["service"]); assert_eq!(retained.software_environments[0].context_ref, "env-0"); } #[test] fn adapter_service_gap_does_not_hide_a_retained_observation() { let mut adapter = crate::ExecutionEvidence::gap("adapter", "fixture"); adapter.append(crate::ExecutionEvidence::gap("service", "report_missing")); let retained: crate::ExecutionEvidence = serde_json::from_value(serde_json::json!({ "software_environments": [{ "context_ref": "original-service", "participants": ["service"], "status": "available", "complete": true, "entries": [{"role": "harness", "kind": "component", "name": "openclaw", "version": {"status": "available", "value": "2026.9.4", "source": "service:startupz", "origin": "reported", "observed_at": "2026-10-27T12:00:00Z"}}] }] })) .unwrap(); let mut report = crate::ExecutionEvidence::gap("core", "fixture"); append_adapter_evidence(&mut report, &adapter, Some(&retained)); assert!(report.valid()); assert_eq!(report.software_environments.len(), 4); assert_eq!( report.software_environments[2].reason.as_deref(), Some("report_missing") ); assert_eq!( report.software_environments[3].entries, retained.software_environments[0].entries ); assert_eq!( retained.software_environments[0].context_ref, "original-service" ); } #[test] fn runtime_binding_excludes_observations_but_binds_initial_collection_control() { let (root, plan) = local_host_plan("ok"); let environment = prepare_environment(&plan).unwrap(); let original = runtime_binding("test-runtime", None, &plan, &environment).unwrap(); let mut changed = plan.clone(); changed.config.provenance.reports = Some(crate::ExecutionEvidence::gap("application", "test")); assert_eq!( runtime_binding("test-runtime", None, &changed, &environment).unwrap(), original ); changed.config.provenance.enabled = false; assert_ne!( runtime_binding("test-runtime", None, &changed, &environment).unwrap(), original ); std::fs::remove_dir_all(root).unwrap(); } #[test] fn explicit_binding_starts_in_the_prepared_local_environment() { let (root, plan) = local_host_plan("success"); let environment = prepare_environment(&plan).expect("prepare local environment"); let runtime = start_runtime_in(&plan, &environment).expect("bind local environment"); assert_eq!( runtime.environment.environment_id, environment.environment_id ); assert_eq!(runtime.environment, environment); stop_runtime(&plan, &runtime).expect("stop runtime"); release_environment(&environment).expect("release environment"); let _ = fs::remove_dir_all(root); } #[test] fn explicit_binding_rejects_a_handle_with_different_paths() { let (root, plan) = local_host_plan("success"); let environment = prepare_environment(&plan).expect("prepare local environment"); for (field, path) in [ ("workspace", root.join("other-workspace")), ("artifacts", root.join("other-artifacts")), ] { let mut changed_plan = plan.clone(); let configured = changed_plan .environment_plan .as_mut() .expect("resolved environment plan"); if field == "workspace" { configured.workspace = Some(path); } else { configured.artifacts = Some(path); } let error = start_runtime_in(&changed_plan, &environment) .expect_err("environment paths must match the plan"); assert!(matches!( error, FabricError::EnvironmentHandleMismatch { field: mismatch_field, .. } if mismatch_field == field )); } let _ = fs::remove_dir_all(root); } #[test] fn environment_release_reports_unsupported_provider_without_adapter_kind() { let (root, plan) = local_host_plan("success"); let mut environment = prepare_environment(&plan).expect("prepare local environment"); environment.provider = "unregistered".to_string(); let error = release_environment(&environment).expect_err("release is unsupported"); assert!(matches!( error, FabricError::UnsupportedEnvironmentRelease { ref provider } if provider == "unregistered" )); assert!(error.to_string().contains("environment release")); let _ = fs::remove_dir_all(root); } #[test] fn explicit_binding_rejects_a_handle_from_another_provider() { let (root, plan) = local_host_plan("success"); let mut environment = prepare_environment(&plan).expect("prepare local environment"); environment.provider = "openshell".to_string(); let error = start_runtime_in(&plan, &environment).expect_err("provider must match"); assert!(matches!( error, FabricError::EnvironmentHandleMismatch { field: "provider", expected, actual, environment_id, } if expected == "local" && actual == "openshell" && environment_id == environment.environment_id )); let _ = fs::remove_dir_all(root); } #[test] fn explicit_binding_does_not_run_a_non_local_environment_on_the_host() { let (root, mut plan) = local_host_plan("success"); let mut environment = prepare_environment(&plan).expect("prepare local environment"); plan.environment_plan .as_mut() .expect("resolved environment plan") .provider = "docker".to_string(); environment.provider = "docker".to_string(); let error = start_runtime_in(&plan, &environment) .expect_err("non-local environment must use its provider runtime"); assert!(matches!( error, FabricError::UnsupportedEnvironmentProvider { provider, adapter_kind: AdapterKind::Python, } if provider == "docker" )); let _ = fs::remove_dir_all(root); } } fn covers_service_observations( adapter: &crate::ExecutionEvidence, retained: &crate::ExecutionEvidence, ) -> bool { // Availability alone does not establish coverage of startup observations. retained.software_environments.iter().all(|old| { adapter.software_environments.iter().any(|current| { current.status == crate::provenance::InventoryStatus::Available && current.participants.iter().any(|p| p == "service") && old.entries.iter().all(|entry| { current.entries.iter().any(|new| { new.role == entry.role && new.kind == entry.kind && new.name == entry.name && (new.version.status == crate::provenance::EvidenceStatus::Available || entry.version.status != crate::provenance::EvidenceStatus::Available) }) }) }) }) && retained.infrastructure.as_ref().is_none_or(|old| { adapter.infrastructure.as_ref().is_some_and(|current| { old.sandbox.as_ref().is_none_or(|old| { current.sandbox.as_ref().is_some_and(|new| { old.provider == new.provider && old.service_version == new.service_version }) }) && old.container.as_ref().is_none_or(|old| { current.container.as_ref().is_some_and(|new| { old.engine == new.engine && old.engine_version == new.engine_version }) }) }) }) } fn append_adapter_evidence( report: &mut crate::ExecutionEvidence, adapter: &crate::ExecutionEvidence, retained_service: Option<&crate::ExecutionEvidence>, ) { let covers_retained = retained_service.is_none_or(|retained| covers_service_observations(adapter, retained)); report.append(adapter.clone()); if !covers_retained && let Some(service) = retained_service { let mut service = service.clone(); let refs: std::collections::BTreeMap<_, _> = service .software_environments .iter() .enumerate() .map(|(index, env)| { ( env.context_ref.clone(), format!("env-{}", report.software_environments.len() + index), ) }) .collect(); let mut sandbox_refs = Vec::new(); let mut container_refs = Vec::new(); // Forwarded startup infrastructure can appear in both reports. Deduplicate // identical owner observations; conflicting reports retain append's gap. if let (Some(old), Some(current)) = (&mut service.infrastructure, &adapter.infrastructure) { if let (Some(left), Some(right)) = (&old.sandbox, ¤t.sandbox) { let mut right = right.clone(); right.context_refs = left.context_refs.clone(); if *left == right { sandbox_refs = left .context_refs .iter() .filter_map(|key| refs.get(key).cloned()) .collect(); old.sandbox = None; } } if let (Some(left), Some(right)) = (&old.container, ¤t.container) { let mut right = right.clone(); right.context_refs = left.context_refs.clone(); if *left == right { container_refs = left .context_refs .iter() .filter_map(|key| refs.get(key).cloned()) .collect(); old.container = None; } } } report.append(service); if !sandbox_refs.is_empty() || !container_refs.is_empty() { let surviving: std::collections::BTreeSet<_> = report .software_environments .iter() .filter(|env| { env.participants .iter() .any(|participant| participant == "service") }) .map(|env| env.context_ref.clone()) .collect(); let extend = |keys: &mut Vec, additional: Vec| { for key in additional { if surviving.contains(&key) && !keys.contains(&key) { keys.push(key); } } }; if let Some(infra) = &mut report.infrastructure { if let Some(value) = &mut infra.sandbox { extend(&mut value.context_refs, sandbox_refs); } if let Some(value) = &mut infra.container { extend(&mut value.context_refs, container_refs); } } // Additional joins also count toward the serialized byte budget. report.append(crate::ExecutionEvidence::default()); } } } fn retain_service_snapshot( current: crate::ExecutionEvidence, retained: Option<&crate::ExecutionEvidence>, ) -> crate::ExecutionEvidence { if let Some(retained) = retained && ((covers_service_observations(retained, ¤t) && current.software_environments.iter().all(|env| { env.entries.iter().all(|entry| { retained .software_environments .iter() .any(|old| old.entries.contains(entry)) }) })) || !current .software_environments .iter() .any(|env| env.status == crate::provenance::InventoryStatus::Available)) { return retained.clone(); } let mut result = crate::ExecutionEvidence::default(); append_adapter_evidence(&mut result, ¤t, retained); result } fn service_evidence(report: &crate::ExecutionEvidence) -> Option { let mut environments: Vec<_> = report .software_environments .iter() .filter(|env| env.participants.iter().any(|p| p == "service")) .cloned() .collect(); if environments.is_empty() { return None; } let mut refs = std::collections::BTreeMap::new(); for (index, env) in environments.iter_mut().enumerate() { let key = format!("env-{index}"); refs.insert(env.context_ref.clone(), key.clone()); env.context_ref = key; } let remap = |keys: &mut Vec| { *keys = keys .iter() .filter_map(|key| refs.get(key).cloned()) .collect(); !keys.is_empty() }; let mut infrastructure = report.infrastructure.clone(); if let Some(infra) = &mut infrastructure { if let Some(value) = &mut infra.sandbox && !remap(&mut value.context_refs) { infra.sandbox = None; } if let Some(value) = &mut infra.container && !remap(&mut value.context_refs) { infra.container = None; } } Some(crate::ExecutionEvidence { software_environments: environments, infrastructure, }) } fn service_metadata_evidence(info: &AdapterServiceInfo) -> Option { use crate::provenance::*; let value = info.metadata.get("openclaw_version")?.as_str()?.to_string(); Some(ExecutionEvidence { software_environments: vec![SoftwareEnvironment { context_ref: "env-0".into(), participants: vec!["service".into()], status: InventoryStatus::Available, complete: true, reason: None, entries: vec![SoftwareEntry { role: "harness".into(), kind: "component".into(), name: "openclaw".into(), version: VersionObservation { status: EvidenceStatus::Available, value: Some(value), source: "core:service_start_report".into(), origin: EvidenceOrigin::Reported, observed_at: crate::time::rfc3339_millis(SystemTime::now()), reason: None, }, }], }], infrastructure: None, }) }