openapi: 3.2.0 info: title: Happyrobot Public Custom Evals API description: Public API endpoints for Happyrobot version: 0.1.1 servers: - url: https://platform.happyrobot.ai/api/v2 security: - bearerAuth: [] tags: - name: Custom Evals paths: /nodes/{node_id}/custom-evals: get: summary: List custom evals for a prompt node tags: - Custom Evals parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: data: type: array items: type: object properties: id: type: string org_id: type: string workflow_id: type: - string - 'null' prompt_node_id: type: string prompt_node_persistent_id: type: - string - 'null' name: type: string description: type: - string - 'null' test_messages: type: array items: type: object additionalProperties: {} expected_response: type: - string - 'null' judge_model: type: - string - 'null' expected_tool_calls: type: - array - 'null' items: type: object additionalProperties: {} variables: type: object additionalProperties: {} source_run_id: type: - string - 'null' is_deleted: type: boolean eval_mode: type: string examples: type: array items: type: object additionalProperties: {} northstar_ids: type: array items: type: string folder_id: type: - string - 'null' created_at: type: string updated_at: type: string required: - id - org_id - workflow_id - prompt_node_id - prompt_node_persistent_id - name - description - test_messages - expected_response - judge_model - expected_tool_calls - variables - source_run_id - is_deleted - eval_mode - examples - northstar_ids - folder_id - created_at - updated_at additionalProperties: false folders: type: array items: type: object properties: id: type: string name: type: string required: - id - name additionalProperties: false required: - data - folders additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false post: summary: Create a custom eval for a prompt node tags: - Custom Evals requestBody: content: application/json: schema: type: object properties: name: type: string minLength: 1 test_messages: type: array items: type: object properties: role: type: string enum: - user - assistant - tool content: type: string turn_index: type: integer minimum: -9007199254740991 maximum: 9007199254740991 tool_calls: type: array items: type: object properties: function: type: object properties: name: type: string arguments: type: string required: - name - arguments id: type: string required: - function tool_call_id: type: string required: - role - content - turn_index description: type: string expected_response: type: string judge_model: default: gpt-5.4-nano-medium type: string expected_tool_calls: type: array items: type: object properties: name: type: string args_description: type: object additionalProperties: type: string required: - name variables: type: object additionalProperties: {} source_run_id: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ eval_mode: type: string enum: - custom - northstar northstar_ids: type: array items: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ required: - name - test_messages required: true parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '201': description: Default Response content: application/json: schema: type: object properties: test: type: object properties: id: type: string org_id: type: string workflow_id: type: - string - 'null' prompt_node_id: type: string prompt_node_persistent_id: type: - string - 'null' name: type: string description: type: - string - 'null' test_messages: type: array items: type: object additionalProperties: {} expected_response: type: - string - 'null' judge_model: type: - string - 'null' expected_tool_calls: type: - array - 'null' items: type: object additionalProperties: {} variables: type: object additionalProperties: {} source_run_id: type: - string - 'null' is_deleted: type: boolean eval_mode: type: string examples: type: array items: type: object additionalProperties: {} northstar_ids: type: array items: type: string folder_id: type: - string - 'null' created_at: type: string updated_at: type: string required: - id - org_id - workflow_id - prompt_node_id - prompt_node_persistent_id - name - description - test_messages - expected_response - judge_model - expected_tool_calls - variables - source_run_id - is_deleted - eval_mode - examples - northstar_ids - folder_id - created_at - updated_at additionalProperties: false required: - test additionalProperties: false '400': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /nodes/{node_id}/custom-evals/tools: get: summary: Get available tools for custom evals tags: - Custom Evals description: Available tool/function definitions for this node — for building expected_tool_calls. parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: tools: type: array items: type: object properties: id: type: string name: type: string function: type: object additionalProperties: {} required: - id - name - function additionalProperties: false required: - tools additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /nodes/{node_id}/custom-evals/default-variables: get: summary: Get default variables for custom evals tags: - Custom Evals description: Merged default variable values available at runtime (org + use-case + workflow levels). parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: query name: version_id required: true - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: variables: type: object additionalProperties: {} required: - variables additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /nodes/{node_id}/custom-evals/extract-from-run: post: summary: Extract custom eval from a run tags: - Custom Evals description: Build a custom eval from an existing run. Auto-extracts messages, variables, and suggests a name. requestBody: content: application/json: schema: type: object properties: run_id: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ message_id: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ required: - run_id - message_id required: true parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: messages: type: array items: type: object additionalProperties: {} variables: type: object additionalProperties: {} suggested_name: type: string source_run_id: type: string navigation_info: type: object properties: prompt_node_id: type: string workflow_id: type: string required: - prompt_node_id - workflow_id additionalProperties: false required: - messages - variables - suggested_name - source_run_id - navigation_info additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /nodes/{node_id}/custom-evals/folders: post: summary: Create a custom eval folder for a prompt node tags: - Custom Evals description: Create a folder for organizing this prompt node's custom evals. File evals into it via folder_id on update. requestBody: content: application/json: schema: type: object properties: name: type: string minLength: 1 required: - name required: true parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: node_id required: true security: - bearerAuth: [] responses: '201': description: Default Response content: application/json: schema: type: object properties: folder: type: object properties: id: type: string name: type: string required: - id - name additionalProperties: false required: - folder additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /custom-evals/{eval_id}: get: summary: Get a custom eval by ID tags: - Custom Evals parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: eval_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: test: type: object properties: id: type: string org_id: type: string workflow_id: type: - string - 'null' prompt_node_id: type: string prompt_node_persistent_id: type: - string - 'null' name: type: string description: type: - string - 'null' test_messages: type: array items: type: object additionalProperties: {} expected_response: type: - string - 'null' judge_model: type: - string - 'null' expected_tool_calls: type: - array - 'null' items: type: object additionalProperties: {} variables: type: object additionalProperties: {} source_run_id: type: - string - 'null' is_deleted: type: boolean eval_mode: type: string examples: type: array items: type: object additionalProperties: {} northstar_ids: type: array items: type: string folder_id: type: - string - 'null' created_at: type: string updated_at: type: string required: - id - org_id - workflow_id - prompt_node_id - prompt_node_persistent_id - name - description - test_messages - expected_response - judge_model - expected_tool_calls - variables - source_run_id - is_deleted - eval_mode - examples - northstar_ids - folder_id - created_at - updated_at additionalProperties: false required: - test additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false patch: summary: Update a custom eval tags: - Custom Evals requestBody: content: application/json: schema: type: object properties: name: type: string minLength: 1 description: type: string test_messages: type: array items: type: object properties: role: type: string enum: - user - assistant - tool content: type: string turn_index: type: integer minimum: -9007199254740991 maximum: 9007199254740991 tool_calls: type: array items: type: object properties: function: type: object properties: name: type: string arguments: type: string required: - name - arguments id: type: string required: - function tool_call_id: type: string required: - role - content - turn_index expected_response: type: string judge_model: type: - string - 'null' expected_tool_calls: type: array items: type: object properties: name: type: string args_description: type: object additionalProperties: type: string required: - name variables: type: object additionalProperties: {} eval_mode: type: string enum: - custom - northstar northstar_ids: type: array items: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ folder_id: type: - string - 'null' format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: eval_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: test: type: object properties: id: type: string org_id: type: string workflow_id: type: - string - 'null' prompt_node_id: type: string prompt_node_persistent_id: type: - string - 'null' name: type: string description: type: - string - 'null' test_messages: type: array items: type: object additionalProperties: {} expected_response: type: - string - 'null' judge_model: type: - string - 'null' expected_tool_calls: type: - array - 'null' items: type: object additionalProperties: {} variables: type: object additionalProperties: {} source_run_id: type: - string - 'null' is_deleted: type: boolean eval_mode: type: string examples: type: array items: type: object additionalProperties: {} northstar_ids: type: array items: type: string folder_id: type: - string - 'null' created_at: type: string updated_at: type: string required: - id - org_id - workflow_id - prompt_node_id - prompt_node_persistent_id - name - description - test_messages - expected_response - judge_model - expected_tool_calls - variables - source_run_id - is_deleted - eval_mode - examples - northstar_ids - folder_id - created_at - updated_at additionalProperties: false required: - test additionalProperties: false '400': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false delete: summary: Delete a custom eval tags: - Custom Evals parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: eval_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: ok: type: boolean enum: - true required: - ok additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /custom-evals/{eval_id}/run: post: summary: Run a custom eval tags: - Custom Evals description: Execute the eval asynchronously. Poll /custom-evals/:eval_id/runs for results. requestBody: content: application/json: schema: type: object properties: version_id: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ required: - version_id required: true parameters: - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: eval_id required: true security: - bearerAuth: [] responses: '202': description: Default Response content: application/json: schema: type: object properties: run_id: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ required: - run_id additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false /custom-evals/{eval_id}/runs: get: summary: List custom eval runs tags: - Custom Evals parameters: - schema: default: 20 type: integer maximum: 100 in: query name: limit required: false - schema: type: string format: uuid pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ in: path name: eval_id required: true security: - bearerAuth: [] responses: '200': description: Default Response content: application/json: schema: type: object properties: runs: type: array items: type: object properties: id: type: string custom_test_id: type: string input_messages: type: array items: type: object additionalProperties: {} output_messages: type: array items: type: object additionalProperties: {} actual_tool_calls: type: - array - 'null' items: type: object additionalProperties: {} passed: type: - boolean - 'null' judge_reasoning: type: - string - 'null' tool_calls_matched: type: - boolean - 'null' version_id: type: - string - 'null' version_name: type: - string - 'null' version_number: type: - number - 'null' created_at: type: string required: - id - custom_test_id - input_messages - output_messages - actual_tool_calls - passed - version_id - created_at additionalProperties: false required: - runs additionalProperties: false '401': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '404': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false '500': description: Default Response content: application/json: schema: type: object properties: error: type: string message: type: string statusCode: type: integer minimum: -9007199254740991 maximum: 9007199254740991 details: {} required: - error additionalProperties: false components: securitySchemes: bearerAuth: type: http scheme: bearer bearerFormat: Opaque