{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/microsoft/waza/main/schemas/eval.schema.json", "title": "Waza Eval Specification", "description": "Schema for scenario, skill and custom-agent evaluation suites.", "type": "object", "required": [ "name", "config", "metrics" ], "additionalProperties": false, "properties": { "name": { "type": "string", "minLength": 1, "description": "Unique name for this evaluation specification." }, "description": { "type": "string", "description": "Detailed description of what this evaluation tests." }, "scenario": { "type": "string", "pattern": "\\S", "description": "Workflow identity, independent of optional skill or custom-agent context. Requires explicit eval schemaVersion 2.0 so older executables reject new discovery semantics." }, "skill": { "type": "string", "minLength": 1, "description": "Optional skill or custom-agent context for a scenario; required for legacy skill suites. Existing discovery precedence is preserved." }, "schemaVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+$", "default": "1.4", "description": "Legacy evals default to 1.4. Scenario evals require explicit 2.0; tasks and results remain 1.x." }, "version": { "type": "string", "minLength": 1, "description": "Version identifier for this evaluation specification." }, "config": { "$ref": "#/$defs/config" }, "hooks": { "$ref": "#/$defs/hooksConfig" }, "mcp_mocks": { "type": "array", "items": { "$ref": "#/$defs/mcpMock" }, "description": "Deterministic MCP mock servers launched for hermetic evals. Requires schemaVersion 1.1 when present." }, "command_mocks": { "type": "array", "items": { "$ref": "#/$defs/commandMock" }, "description": "Deterministic executable shims for CLI-dependent evals. Requires schemaVersion 1.3 or newer and executor copilot-sdk." }, "inputs": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Global input variables available to all tasks." }, "tasks_from": { "type": "string", "description": "CSV dataset path, resolved relative to the eval directory unless absolute. When nonempty, takes precedence over tasks." }, "range": { "type": "array", "items": { "type": "integer", "minimum": 0 }, "minItems": 2, "maxItems": 2, "description": "Task index range [start, end] to run a subset of tasks." }, "graders": { "type": "array", "items": { "$ref": "#/$defs/graderConfig" }, "description": "Global graders applied to all tasks in this evaluation." }, "metrics": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/metric" }, "description": "Metrics that define how overall skill quality is measured." }, "tasks": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Glob patterns for task YAML files (e.g., 'tasks/*.yaml')." }, "baseline": { "type": "boolean", "default": false, "description": "When true, marks this as a baseline run for comparison." } }, "allOf": [ { "if": { "not": { "required": ["schemaVersion"], "properties": { "schemaVersion": { "const": "2.0" } } } }, "then": { "properties": { "command_mocks": { "items": { "properties": { "responses": { "items": { "not": { "required": ["sequence"] } } } } } }, "mcp_mocks": { "items": { "properties": { "tools": { "additionalProperties": { "properties": { "responses": { "items": { "not": { "required": ["sequence"] } } } } } } } } } } } }, { "anyOf": [ { "required": ["skill"] }, { "required": ["scenario"] } ] }, { "if": { "required": ["scenario"] }, "then": { "required": ["schemaVersion"], "properties": { "schemaVersion": { "const": "2.0" } } } }, { "if": { "required": ["schemaVersion"], "properties": { "schemaVersion": { "const": "2.0" } } }, "then": { "required": ["scenario"] } }, { "anyOf": [ { "required": ["tasks_from"], "properties": {"tasks_from": {"minLength": 1}} }, { "required": ["tasks"], "properties": {"tasks": {"minItems": 1}} } ] }, { "if": { "required": ["config"], "properties": { "config": { "required": ["executor"], "properties": { "executor": { "not": { "const": "copilot-sdk" } } } } } }, "then": { "properties": { "graders": { "items": { "properties": { "config": { "not": { "anyOf": [ {"required": ["reasoning_effort"]}, {"required": ["evidence_context"]} ] } } } } } } } }, { "if": { "required": ["command_mocks"] }, "then": { "properties": { "schemaVersion": { "pattern": "^(?:0*1\\.0*(?:[3-9]|[1-9][0-9]+)|0*(?:[2-9]|[1-9][0-9]+)\\.[0-9]+)$" }, "config": { "properties": { "executor": { "const": "copilot-sdk" } } } } } } ], "$defs": { "config": { "type": "object", "required": [ "trials_per_task", "timeout_seconds", "executor", "model" ], "additionalProperties": false, "description": "Execution configuration for the evaluation.", "properties": { "trials_per_task": { "type": "integer", "minimum": 1, "description": "Number of times to run each task (for consistency measurement)." }, "timeout_seconds": { "type": "integer", "minimum": 1, "description": "Maximum time in seconds for each task execution." }, "first_event_timeout_seconds": { "type": "integer", "minimum": 0, "default": 0, "description": "Maximum time in seconds to wait for the first session event before treating a run as a session-start hang and aborting it. Intended to be much shorter than timeout_seconds, which must cover the slowest legitimate full turn. 0 (the default) disables the check." }, "parallel": { "type": "boolean", "default": false, "description": "When true, run tasks in parallel." }, "workers": { "type": "integer", "minimum": 0, "description": "Maximum number of concurrent workers when parallel is true. Use 0 or omit to auto-size workers." }, "fail_fast": { "type": "boolean", "default": false, "description": "When true, stop the evaluation on the first task error." }, "executor": { "type": "string", "enum": [ "copilot-sdk", "mock" ], "description": "Execution engine to use. 'copilot-sdk' for real evaluations, 'mock' for testing." }, "model": { "type": "string", "minLength": 1, "description": "Default model identifier for evaluations.", "examples": [ "gpt-4o", "claude-sonnet-4-20250514", "gpt-4o-mini" ] }, "reasoning_effort": { "type": "string", "enum": ["low", "medium", "high", "xhigh", "max"], "description": "Reasoning effort for copilot-sdk task and responder sessions." }, "max_attempts": { "type": "integer", "minimum": 1, "description": "Maximum retry attempts for failed task executions." }, "group_by": { "type": "string", "enum": ["model", "tags"], "description": "Group results by model or by each task's complete ordered tag set." }, "judge_model": { "type": "string", "description": "Separate model to use for prompt-based grading (overrides the default model for prompt graders)." }, "judge_reasoning_effort": { "type": "string", "enum": ["low", "medium", "high", "xhigh", "max"], "description": "Default reasoning effort for copilot-sdk prompt-grader sessions." }, "skill_directories": { "type": "array", "items": { "type": "string" }, "description": "Additional directories to search for skill definitions." }, "instruction_files": { "type": "array", "items": { "type": "string" }, "description": "Instruction files to apply to every task, resolved relative to the active context/fixtures directory." }, "inject_skill_body": { "type": "boolean", "default": true, "description": "When false, do not inject the target SKILL.md or .agent.md body into the system prompt. Copilot SDK skill discovery remains enabled, but Waza does not emit a synthetic available_skills summary." }, "trigger_skill_routing": { "type": "boolean", "default": false, "description": "When true with inject_skill_body=false, append an eval-only routing instruction that asks the agent to invoke the target skill when the task is in scope. This preserves ordinary eval behavior unless explicitly enabled." }, "disabled_skills": { "type": "array", "items": { "type": "string" }, "description": "Skills to disable. Use [\"*\"] to disable all skills, or list specific skill directory names to exclude." }, "required_skills": { "type": "array", "items": { "type": "string" }, "description": "Skill names that must be available for this evaluation to run." }, "mcp_servers": { "type": "object", "additionalProperties": true, "description": "MCP server configurations keyed by server name." } }, "allOf": [ { "if": { "anyOf": [ { "required": ["reasoning_effort"] }, { "required": ["judge_reasoning_effort"] } ] }, "then": { "properties": { "executor": { "const": "copilot-sdk" } } } } ] }, "mcpMock": { "type": "object", "required": [ "name" ], "additionalProperties": false, "description": "A deterministic MCP mock server backed by inline tools and/or JSON fixture files.", "anyOf": [ { "required": [ "fixtures" ] }, { "required": [ "tools" ] } ], "properties": { "name": { "type": "string", "minLength": 1, "description": "MCP server name exposed to the agent." }, "fixtures": { "type": "string", "minLength": 1, "description": "Directory containing JSON fixture files, resolved relative to eval.yaml. Each file can define one tool or a {\"tools\": {...}} bundle." }, "tools": { "type": "object", "additionalProperties": { "$ref": "#/$defs/mcpMockTool" }, "description": "Inline mocked tools keyed by MCP tool name." } } }, "commandMock": { "type": "object", "required": ["name", "responses"], "additionalProperties": false, "properties": { "name": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", "description": "Executable name to intercept (for example, az or gh)." }, "expect_calls": { "type": "integer", "minimum": 0, "description": "Exact number of invocations expected during this task." }, "responses": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/commandMockResponse" }, "description": "Ordered responses; the first matching argument and environment rule is used." } } }, "commandMockResponse": { "type": "object", "additionalProperties": false, "properties": { "args": { "type": "array", "items": { "type": "string" }, "description": "Exact argument vector, excluding the executable name." }, "args_regex": { "type": "array", "items": { "type": "string" }, "description": "Argument vector matched positionally using full-string regular expressions." }, "environment": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Environment variables that must exactly match for this response." }, "workdir": { "type": "string", "description": "Workspace-relative working directory required for this response." }, "stdout": { "description": "Output written to stdout. Structured values are emitted as JSON." }, "stderr": { "type": "string", "description": "Output written to stderr." }, "exit_code": { "type": "integer", "minimum": 0, "maximum": 255, "default": 0 }, "fixture": { "type": "string", "minLength": 1, "description": "Relative path to a file whose contents are written to stdout." }, "sequence": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "$ref": "#/$defs/commandMockStep" }, "description": "Finite per-matcher outputs consumed once per call; scenario eval schema 2.0 only." } }, "oneOf": [ { "required": ["args"] }, { "required": ["args_regex"] } ], "allOf": [ { "not": { "required": ["args", "args_regex"] } }, { "not": { "required": ["stdout", "fixture"] } }, { "if": { "required": ["sequence"] }, "then": { "not": { "anyOf": [ { "required": ["stdout"] }, { "required": ["stderr"] }, { "required": ["exit_code"] }, { "required": ["fixture"] } ] } } } ] }, "commandMockStep": { "type": "object", "additionalProperties": false, "anyOf": [ { "required": ["stdout"] }, { "required": ["stderr"] }, { "required": ["exit_code"] }, { "required": ["fixture"] } ], "properties": { "stdout": {}, "stderr": { "type": "string" }, "exit_code": { "type": "integer", "minimum": 0, "maximum": 255 }, "fixture": { "type": "string", "minLength": 1 }, "delay_ms": { "type": "integer", "minimum": 0, "maximum": 300000 } }, "not": { "required": ["stdout", "fixture"] } }, "mcpMockTool": { "type": "object", "required": [ "responses" ], "additionalProperties": false, "properties": { "description": { "type": "string", "description": "Tool description returned from tools/list." }, "input_schema": { "type": ["object", "null"], "description": "JSON Schema input schema returned from tools/list. Omit it or use null for the default tool schema." }, "responses": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/mcpMockResponse" }, "description": "Ordered fixture responses. The first matching response is returned." } } }, "mcpMockResponse": { "type": "object", "additionalProperties": false, "anyOf": [ { "required": [ "return" ] }, { "required": [ "error" ] }, { "required": [ "sequence" ] } ], "allOf": [ { "if": { "required": ["sequence"] }, "then": { "not": { "anyOf": [ { "required": ["return"] }, { "required": ["error"] } ] } } } ], "properties": { "match": { "type": "object", "description": "Exact full-arguments match for this response." }, "match_schema": { "type": "object", "description": "JSON Schema that call arguments must satisfy." }, "match_regex": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Per-field regular expressions matched against stringified argument values." }, "return": { "description": "JSON value returned by the mocked tool." }, "error": { "type": "string", "description": "Fixture error returned by the mocked tool." }, "sequence": { "type": "array", "minItems": 1, "maxItems": 100, "items": { "$ref": "#/$defs/mcpMockStep" }, "description": "Finite per-matcher results consumed once per call; scenario eval schema 2.0 only." } } }, "mcpMockStep": { "type": "object", "additionalProperties": false, "oneOf": [ { "required": ["return"] }, { "required": ["error"] } ], "properties": { "return": {}, "error": { "type": "string", "minLength": 1 }, "delay_ms": { "type": "integer", "minimum": 0, "maximum": 300000 } } }, "hooksConfig": { "type": "object", "additionalProperties": false, "description": "Lifecycle hooks that run commands at various evaluation stages.", "properties": { "before_run": { "type": "array", "items": { "$ref": "#/$defs/hookConfig" }, "description": "Commands to run before the entire evaluation starts." }, "after_run": { "type": "array", "items": { "$ref": "#/$defs/hookConfig" }, "description": "Commands to run after the entire evaluation completes." }, "before_task": { "type": "array", "items": { "$ref": "#/$defs/hookConfig" }, "description": "Commands to run before each task execution." }, "after_task": { "type": "array", "items": { "$ref": "#/$defs/hookConfig" }, "description": "Commands to run after each task execution." } } }, "hookConfig": { "type": "object", "required": [ "command" ], "additionalProperties": false, "description": "A single hook command configuration.", "properties": { "command": { "type": "string", "minLength": 1, "description": "The shell command to execute." }, "working_directory": { "type": "string", "description": "Working directory for the command. Defaults to the eval file's directory." }, "exit_codes": { "type": "array", "items": { "type": "integer" }, "description": "Acceptable exit codes. Defaults to [0] if not specified." }, "error_on_fail": { "type": "boolean", "default": false, "description": "When true, a non-acceptable exit code causes the evaluation to fail. When false, a warning is logged and execution continues." } } }, "metric": { "type": "object", "required": [ "name", "weight", "threshold" ], "additionalProperties": false, "description": "Defines a measurement dimension for the evaluation.", "properties": { "name": { "type": "string", "minLength": 1, "description": "Unique identifier for this metric." }, "weight": { "type": "number", "exclusiveMinimum": 0, "description": "Relative contribution weight of this metric to the overall score." }, "threshold": { "type": "number", "minimum": 0, "maximum": 1, "description": "Minimum score (0-1) required for this metric to pass." }, "description": { "type": "string", "description": "Human-readable description of what this metric measures." }, "enabled": { "type": "boolean", "default": true, "description": "When false, this metric is skipped during evaluation." } } }, "graderConfig": { "type": "object", "additionalProperties": false, "description": "Configuration for a grader that scores agent output.", "anyOf": [ { "required": [ "ref" ] }, { "required": [ "type", "name" ] } ], "properties": { "ref": { "type": "string", "minLength": 1, "description": "Go-module-style remote grader reference resolved by waza get." }, "type": { "type": "string", "enum": [ "code", "prompt", "text", "file", "json_schema", "program", "behavior", "action_sequence", "skill_invocation", "trigger", "diff", "tool_constraint", "tool_calls" ], "description": "The grader type." }, "name": { "type": "string", "minLength": 1, "description": "Unique identifier for this grader." }, "script": { "type": "string", "description": "Path to an external grading script." }, "rubric": { "type": "string", "description": "Evaluation rubric text (used by the prompt grader)." }, "model": { "type": "string", "description": "Override the default model for this grader." }, "weight": { "type": "number", "exclusiveMinimum": 0, "default": 1.0, "description": "Contribution weight of this grader. Defaults to 1.0." }, "config": { "type": "object", "description": "Type-specific configuration for this grader." } }, "allOf": [ { "if": { "properties": {"type": {"const": "tool_calls"}}, "required": ["type"] }, "then": { "properties": {"config": {"$ref": "#/$defs/toolCallsGraderConfig"}} } }, { "if": { "properties": { "type": { "const": "code" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/codeGraderConfig" } } } }, { "if": { "properties": { "type": { "const": "text" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/textGraderConfig" } } } }, { "if": { "properties": { "type": { "const": "file" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/fileGraderConfig" } } } }, { "if": { "properties": { "type": { "const": "behavior" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/behaviorGraderConfig" } } } }, { "if": { "properties": { "type": { "const": "action_sequence" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/actionSequenceGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "skill_invocation" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/skillInvocationGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "tool_constraint" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/toolConstraintGraderConfig" } } } }, { "if": { "properties": { "type": { "const": "trigger" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/triggerGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "prompt" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/promptGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "json_schema" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/jsonSchemaGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "program" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/programGraderConfig" } }, "required": [ "config" ] } }, { "if": { "properties": { "type": { "const": "diff" } }, "required": [ "type" ] }, "then": { "properties": { "config": { "$ref": "#/$defs/diffGraderConfig" } }, "required": [ "config" ] } } ] }, "codeGraderConfig": { "type": "object", "required": [ "assertions" ], "additionalProperties": false, "description": "Config for the code (inline script) grader. Evaluates assertions against agent output using Python or JavaScript.", "properties": { "assertions": { "type": "array", "minItems": 1, "items": { "type": "string" }, "description": "Python or JavaScript assertions to evaluate. Available variables: output, outcome, transcript, tool_calls, errors, duration_ms." }, "language": { "type": "string", "enum": [ "python", "javascript" ], "default": "python", "description": "Language for the assertion expressions. Defaults to 'python'." } } }, "textGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the text grader. Validates agent output using substring matching and regex patterns.", "anyOf": [ { "required": [ "contains" ] }, { "required": [ "not_contains" ] }, { "required": [ "contains_cs" ] }, { "required": [ "not_contains_cs" ] }, { "required": [ "regex_match" ] }, { "required": [ "regex_not_match" ] } ], "properties": { "contains": { "type": "array", "items": { "type": "string" }, "description": "Substrings that MUST appear in the output (case-insensitive)." }, "not_contains": { "type": "array", "items": { "type": "string" }, "description": "Substrings that MUST NOT appear in the output (case-insensitive)." }, "contains_cs": { "type": "array", "items": { "type": "string" }, "description": "Substrings that MUST appear in the output (case-sensitive)." }, "not_contains_cs": { "type": "array", "items": { "type": "string" }, "description": "Substrings that MUST NOT appear in the output (case-sensitive)." }, "regex_match": { "type": "array", "items": { "type": "string" }, "description": "Regex patterns that MUST match somewhere in the output." }, "regex_not_match": { "type": "array", "items": { "type": "string" }, "description": "Regex patterns that MUST NOT match anywhere in the output." } } }, "fileGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the file grader. Validates file existence and content in the workspace.", "anyOf": [ { "required": [ "must_exist" ] }, { "required": [ "must_not_exist" ] }, { "required": [ "content_patterns" ] } ], "properties": { "must_exist": { "type": "array", "items": { "type": "string" }, "description": "Workspace-relative file paths that must exist after execution." }, "must_not_exist": { "type": "array", "items": { "type": "string" }, "description": "Workspace-relative file paths that must NOT exist after execution." }, "content_patterns": { "type": "array", "items": { "$ref": "#/$defs/fileContentPattern" }, "description": "Content validation rules for specific files." } } }, "fileContentPattern": { "type": "object", "required": [ "path" ], "additionalProperties": false, "description": "Content validation rules for a specific file.", "anyOf": [ { "required": [ "path", "must_match" ] }, { "required": [ "path", "must_not_match" ] } ], "properties": { "path": { "type": "string", "minLength": 1, "description": "Workspace-relative file path to validate." }, "must_match": { "type": "array", "items": { "type": "string" }, "description": "Regex patterns that must match in the file content." }, "must_not_match": { "type": "array", "items": { "type": "string" }, "description": "Regex patterns that must NOT match in the file content." } } }, "behaviorGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the behavior grader. Validates agent behavior constraints like tool usage and efficiency.", "anyOf": [ { "required": [ "max_tool_calls" ] }, { "required": [ "max_tokens" ] }, { "required": [ "required_tools" ] }, { "required": [ "forbidden_tools" ] }, { "required": [ "max_duration_ms" ] } ], "properties": { "max_tool_calls": { "type": "integer", "minimum": 0, "description": "Maximum number of tool invocations allowed." }, "max_tokens": { "type": "integer", "minimum": 0, "description": "Maximum total token usage allowed." }, "required_tools": { "type": "array", "items": { "type": "string" }, "description": "Tools that MUST be used during execution." }, "forbidden_tools": { "type": "array", "items": { "type": "string" }, "description": "Tools that MUST NOT be used during execution." }, "max_duration_ms": { "type": "integer", "minimum": 0, "description": "Maximum execution duration in milliseconds." } } }, "actionSequenceGraderConfig": { "type": "object", "required": [ "matching_mode", "expected_actions" ], "additionalProperties": false, "description": "Config for the action sequence grader. Validates the sequence of tool calls made by the agent.", "properties": { "matching_mode": { "type": "string", "enum": [ "exact_match", "in_order_match", "any_order_match" ], "description": "How to compare actual tool calls against expected. 'exact_match': identical sequence. 'in_order_match': all expected actions appear in order (extras allowed). 'any_order_match': all expected actions appear in any order." }, "expected_actions": { "type": "array", "minItems": 1, "items": { "type": "string" }, "description": "Expected tool names in sequence." } } }, "skillInvocationGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the skill invocation grader. Validates that specific skills were invoked during execution.", "anyOf": [ { "required": [ "required_skills" ], "properties": { "required_skills": { "minItems": 1 } } }, { "required": [ "forbidden_skills" ], "properties": { "forbidden_skills": { "minItems": 1 } } } ], "allOf": [ { "if": { "required": [ "required_skills" ], "properties": { "required_skills": { "minItems": 1 } } }, "then": { "required": [ "mode" ] } } ], "properties": { "mode": { "type": "string", "enum": [ "exact_match", "in_order", "any_order" ], "description": "How to compare actual skill invocations against expected required skills. Required when required_skills is set; ignored for forbidden-only configs." }, "required_skills": { "type": "array", "items": { "type": "string" }, "description": "Skill names that must be invoked." }, "forbidden_skills": { "type": "array", "items": { "type": "string" }, "description": "Skill names that must not be invoked." }, "allow_extra": { "type": "boolean", "default": true, "description": "When true, extra skill invocations beyond the required list are allowed. Ignored for forbidden-only configs." } } }, "triggerGraderConfig": { "type": "object", "required": [ "skill_path", "mode" ], "additionalProperties": false, "description": "Config for the trigger heuristic grader. Scores whether a prompt should activate a skill.", "properties": { "skill_path": { "type": "string", "minLength": 1, "description": "Path to the skill SKILL.md used for keyword extraction." }, "mode": { "type": "string", "enum": [ "positive", "negative" ], "description": "positive expects trigger relevance >= threshold; negative expects relevance < threshold." }, "threshold": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.6, "description": "Relevance threshold between 0 and 1." } } }, "toolArgumentMatchers": { "type": "object", "description": "Argument-name to matcher mapping. Each matcher must define exactly one of equals, regex, contains, range, or json_schema.", "additionalProperties": { "type": "object", "minProperties": 1, "maxProperties": 1, "additionalProperties": false, "properties": { "equals": {}, "regex": { "type": "string", "minLength": 1 }, "contains": { "type": "string", "minLength": 1 }, "range": { "type": "object", "minProperties": 1, "additionalProperties": false, "properties": { "gte": { "type": "number" }, "lte": { "type": "number" }, "gt": { "type": "number" }, "lt": { "type": "number" } } }, "json_schema": { "type": "object", "minProperties": 1 } } } }, "toolCallsGraderConfig": { "type": "object", "additionalProperties": false, "anyOf": [ {"required": ["required_tools"]}, {"required": ["forbidden_tools"]}, {"required": ["min_calls"]}, {"required": ["max_calls"]}, {"required": ["expect"]} ], "properties": { "required_tools": {"type": "array", "items": {"type": "string"}}, "forbidden_tools": {"type": "array", "items": {"type": "string"}}, "min_calls": {"type": "integer", "minimum": 0}, "max_calls": {"type": "integer", "minimum": 0}, "expect": { "type": "array", "items": { "type": "object", "required": ["tool"], "additionalProperties": false, "properties": { "tool": {"type": "string", "minLength": 1}, "args": {"$ref": "#/$defs/toolArgumentMatchers"} } } } } }, "toolConstraintGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the tool constraint grader. Validates tool usage constraints.", "anyOf": [ { "required": [ "expect_tools" ] }, { "required": [ "reject_tools" ] }, { "required": [ "allow_only" ] } ], "properties": { "expect_tools": { "type": "array", "items": { "type": "object", "required": [ "tool" ], "additionalProperties": false, "properties": { "tool": { "type": "string", "minLength": 1, "description": "Regex matched against tool name (case-insensitive)." }, "command_pattern": { "type": "string", "description": "Optional regex matched against serialized tool arguments (case-insensitive)." }, "skill_pattern": { "type": "string", "description": "Optional regex matched against the skill argument (case-insensitive)." }, "path_pattern": { "type": "string", "description": "Optional regex matched against the path argument (case-insensitive)." }, "args": { "$ref": "#/$defs/toolArgumentMatchers" } } }, "description": "Tool specs that MUST be used during execution." }, "reject_tools": { "type": "array", "items": { "type": "object", "required": [ "tool" ], "additionalProperties": false, "properties": { "tool": { "type": "string", "minLength": 1, "description": "Regex matched against tool name (case-insensitive)." }, "command_pattern": { "type": "string", "description": "Optional regex matched against serialized tool arguments (case-insensitive)." }, "skill_pattern": { "type": "string", "description": "Optional regex matched against the skill argument (case-insensitive)." }, "path_pattern": { "type": "string", "description": "Optional regex matched against the path argument (case-insensitive)." }, "args": { "$ref": "#/$defs/toolArgumentMatchers" } } }, "description": "Tool specs that MUST NOT be used during execution." }, "allow_only": { "type": "array", "items": { "type": "object", "required": [ "tool" ], "additionalProperties": false, "properties": { "tool": { "type": "string", "minLength": 1, "description": "Exact tool name (case-insensitive). NOT a regex — an entry 'bash' does not match 'bashful'." }, "command_pattern": { "type": "string", "description": "Optional regex matched against serialized tool arguments (case-insensitive)." }, "skill_pattern": { "type": "string", "description": "Optional regex matched against the skill argument (case-insensitive)." }, "path_pattern": { "type": "string", "description": "Optional regex matched against the path argument (case-insensitive)." }, "args": { "$ref": "#/$defs/toolArgumentMatchers" } } }, "description": "Allow-list of tool specs (upper bound). Any observed tool call that does not match at least one entry is a policy violation. Declared tools do NOT need to be exercised. An empty array denies all tool use. This is the field that .agent.md 'tools:' frontmatter is desugared into." } } }, "promptGraderConfig": { "type": "object", "allOf": [{ "if": {"required": ["evidence_context"]}, "then": {"properties": { "continue_session": {"const": false}, "mode": {"const": "independent"} }} }], "anyOf": [ {"required": ["prompt"]}, {"required": ["rubric"]} ], "additionalProperties": false, "description": "Config for the prompt grader. Uses an LLM to evaluate agent output against a rubric or prompt.", "properties": { "evidence_context": {"$ref": "#/$defs/promptEvidenceContext"}, "prompt": { "type": "string", "minLength": 1, "description": "Grading prompt/instructions sent to the judge LLM." }, "rubric": { "type": "string", "minLength": 1, "description": "Existing built-in rubric name or local markdown rubric path." }, "model": { "type": "string", "description": "Override the judge model for this grader." }, "reasoning_effort": { "type": "string", "enum": ["low", "medium", "high", "xhigh", "max"], "description": "Override the default reasoning effort for this judge session." }, "continue_session": { "type": "boolean", "default": false, "description": "When true, resume the original agent session for grading." }, "mode": { "type": "string", "enum": [ "independent", "pairwise" ], "default": "independent", "description": "'independent': grade output alone. 'pairwise': compare baseline vs skill output with position bias detection." } } }, "promptEvidenceContext": { "type": "object", "additionalProperties": false, "description": "Required frozen actor evidence for a fresh isolated final prompt judge. Never resumes an actor or exposes its workspace.", "properties": { "files": {"$ref": "#/$defs/judgeEvidencePaths"}, "evaluator_only": {"$ref": "#/$defs/judgeEvidencePaths"} } }, "judgeEvidencePaths": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 1024, "allOf": [ {"not": {"pattern": "[\\\\:*?\\[\\]\\x00]"}}, {"not": {"pattern": "(^/|/$|//|(^|/)\\.{1,2}(/|$))"}} ] } }, "jsonSchemaGraderConfig": { "type": "object", "additionalProperties": false, "description": "Config for the JSON schema grader. Validates that agent output is valid JSON conforming to a schema.", "anyOf": [ { "required": [ "schema" ] }, { "required": [ "schema_file" ] } ], "properties": { "schema": { "type": "object", "description": "Inline JSON Schema object used for validation." }, "schema_file": { "type": "string", "description": "Path to a JSON Schema file used for validation." }, "extract_json": { "type": "boolean", "default": false, "description": "Extract exactly one JSON object or array from prose or a Markdown JSON code block before validation." } } }, "programGraderConfig": { "type": "object", "required": [ "command" ], "additionalProperties": false, "description": "Config for the program grader. Runs an external program to grade agent output. Output is passed via stdin; exit code 0 = pass.", "properties": { "command": { "type": "string", "minLength": 1, "description": "Path to the program or script to execute." }, "args": { "type": "array", "items": { "type": "string" }, "description": "Arguments to pass to the program." }, "timeout": { "type": "integer", "minimum": 1, "default": 30, "description": "Maximum execution time in seconds. Defaults to 30." } } }, "diffGraderConfig": { "type": "object", "required": [ "expected_files" ], "additionalProperties": false, "description": "Config for the diff grader. Compares workspace files against expected snapshots and content fragments.", "properties": { "expected_files": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/expectedFile" }, "description": "File expectations to validate against the workspace." }, "context_dir": { "type": "string", "description": "Base directory for resolving snapshot paths." }, "update_snapshots": { "type": "boolean", "default": false, "description": "When true, create or overwrite snapshot files with actual workspace content." } } }, "expectedFile": { "type": "object", "required": [ "path" ], "additionalProperties": false, "description": "A single file expectation for the diff grader. At least one of 'snapshot' or 'contains' must be specified.", "anyOf": [ { "required": [ "path", "snapshot" ] }, { "required": [ "path", "contains" ] } ], "properties": { "path": { "type": "string", "minLength": 1, "description": "Workspace-relative path to the file being checked." }, "snapshot": { "type": "string", "description": "Path (relative to context/fixtures dir) of the expected file content for exact match." }, "contains": { "type": "array", "items": { "type": "string" }, "description": "Line fragments that must appear in the file. Prefix with '+' for must-present, '-' for must-absent." } } } } }