openapi: 3.0.0 info: title: OpenAI Assistants Evals API description: The Assistants API allows you to build AI assistants within your own applications. An Assistant has instructions and can leverage models, tools, and knowledge to respond to user queries. The Assistants API currently supports three types of tools - Code Interpreter, Retrieval, and Function calling. In the future, we plan to release more OpenAI-built tools, and allow you to provide your own tools on our platform. version: 2.0.0 termsOfService: https://openai.com/policies/terms-of-use contact: name: OpenAI Support url: https://help.openai.com/ license: name: MIT url: https://github.com/openai/openai-openapi/blob/master/LICENSE servers: - url: https://api.openai.com/v1 security: - ApiKeyAuth: [] tags: - name: Evals description: Manage and run evals in the OpenAI platform. paths: /evals: get: operationId: listEvals tags: - Evals summary: 'List evaluations for a project. ' parameters: - name: after in: query description: Identifier for the last eval from the previous pagination request. required: false schema: type: string - name: limit in: query description: Number of evals to retrieve. required: false schema: type: integer default: 20 - name: order in: query description: Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for descending order. required: false schema: type: string enum: - asc - desc default: asc - name: order_by in: query description: 'Evals can be ordered by creation time or last updated time. Use `created_at` for creation time or `updated_at` for last updated time. ' required: false schema: type: string enum: - created_at - updated_at default: created_at responses: '200': description: A list of evals content: application/json: schema: $ref: '#/components/schemas/EvalList' x-oaiMeta: name: List evals group: evals path: list examples: request: curl: "curl https://api.openai.com/v1/evals?limit=1 \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\npage = client.evals.list()\npage = page.data[0]\nprint(page.id)" javascript: 'import OpenAI from "openai"; const openai = new OpenAI(); const evals = await openai.evals.list({ limit: 1 }); console.log(evals); ' node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const evalListResponse of client.evals.list()) {\n console.log(evalListResponse.id);\n}" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.EvalListPage;\nimport com.openai.models.evals.EvalListParams;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n EvalListPage page = client.evals().list();\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") page = openai.evals.list puts(page)' response: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"object\": \"eval\",\n \"data_source_config\": {\n \"type\": \"stored_completions\",\n \"metadata\": {\n \"usecase\": \"push_notifications_summarizer\"\n },\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\"\n },\n \"sample\": {\n \"type\": \"object\"\n }\n },\n \"required\": [\n \"item\",\n \"sample\"\n ]\n }\n },\n \"testing_criteria\": [\n {\n \"name\": \"Push Notification Summary Grader\",\n \"id\": \"Push Notification Summary Grader-9b876f24-4762-4be9-aff4-db7a9b31c673\",\n \"type\": \"label_model\",\n \"model\": \"o3-mini\",\n \"input\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"\\nLabel the following push notification summary as either correct or incorrect.\\nThe push notification and the summary will be provided below.\\nA good push notificiation summary is concise and snappy.\\nIf it is good, then label it as correct, if not, then incorrect.\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"\\nPush notifications: {{item.input}}\\nSummary: {{sample.output_text}}\\n\"\n }\n }\n ],\n \"passing_labels\": [\n \"correct\"\n ],\n \"labels\": [\n \"correct\",\n \"incorrect\"\n ],\n \"sampling_params\": null\n }\n ],\n \"name\": \"Push Notification Summary Grader\",\n \"created_at\": 1739314509,\n \"metadata\": {\n \"description\": \"A stored completions eval for push notification summaries\"\n }\n }\n ],\n \"first_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"last_id\": \"eval_67aa884cf6688190b58f657d4441c8b7\",\n \"has_more\": true\n}\n" post: operationId: createEval tags: - Evals summary: 'Create the structure of an evaluation that can be used to test a model''s performance. An evaluation is a set of testing criteria and the config for a data source, which dictates the schema of the data used in the evaluation. After creating an evaluation, you can run it on different models and model parameters. We support several types of graders and datasources. For more information, see the [Evals guide](/docs/guides/evals). ' requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/CreateEvalRequest' responses: '201': description: OK content: application/json: schema: $ref: '#/components/schemas/Eval' x-oaiMeta: name: Create eval group: evals path: post examples: request: curl: "curl https://api.openai.com/v1/evals \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"Sentiment\",\n \"data_source_config\": {\n \"type\": \"stored_completions\",\n \"metadata\": {\n \"usecase\": \"chatbot\"\n }\n },\n \"testing_criteria\": [\n {\n \"type\": \"label_model\",\n \"model\": \"o3-mini\",\n \"input\": [\n {\n \"role\": \"developer\",\n \"content\": \"Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Statement: {{item.input}}\"\n }\n ],\n \"passing_labels\": [\n \"positive\"\n ],\n \"labels\": [\n \"positive\",\n \"neutral\",\n \"negative\"\n ],\n \"name\": \"Example label grader\"\n }\n ]\n }'\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\neval = client.evals.create(\n data_source_config={\n \"item_schema\": {\n \"foo\": \"bar\"\n },\n \"type\": \"custom\",\n },\n testing_criteria=[{\n \"input\": [{\n \"content\": \"content\",\n \"role\": \"role\",\n }],\n \"labels\": [\"string\"],\n \"model\": \"model\",\n \"name\": \"name\",\n \"passing_labels\": [\"string\"],\n \"type\": \"label_model\",\n }],\n)\nprint(eval.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst evalObj = await openai.evals.create({\n name: \"Sentiment\",\n data_source_config: {\n type: \"stored_completions\",\n metadata: { usecase: \"chatbot\" }\n },\n testing_criteria: [\n {\n type: \"label_model\",\n model: \"o3-mini\",\n input: [\n { role: \"developer\", content: \"Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'\" },\n { role: \"user\", content: \"Statement: {{item.input}}\" }\n ],\n passing_labels: [\"positive\"],\n labels: [\"positive\", \"neutral\", \"negative\"],\n name: \"Example label grader\"\n }\n ]\n});\nconsole.log(evalObj);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst _eval = await client.evals.create({\n data_source_config: {\n item_schema: { foo: 'bar' },\n type: 'custom',\n },\n testing_criteria: [\n {\n input: [{ content: 'content', role: 'role' }],\n labels: ['string'],\n model: 'model',\n name: 'name',\n passing_labels: ['string'],\n type: 'label_model',\n },\n ],\n});\n\nconsole.log(_eval.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.core.JsonValue;\nimport com.openai.models.evals.EvalCreateParams;\nimport com.openai.models.evals.EvalCreateResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n EvalCreateParams params = EvalCreateParams.builder()\n .customDataSourceConfig(EvalCreateParams.DataSourceConfig.Custom.ItemSchema.builder()\n .putAdditionalProperty(\"foo\", JsonValue.from(\"bar\"))\n .build())\n .addTestingCriterion(EvalCreateParams.TestingCriterion.LabelModel.builder()\n .addInput(EvalCreateParams.TestingCriterion.LabelModel.Input.SimpleInputMessage.builder()\n .content(\"content\")\n .role(\"role\")\n .build())\n .addLabel(\"string\")\n .model(\"model\")\n .name(\"name\")\n .addPassingLabel(\"string\")\n .build())\n .build();\n EvalCreateResponse eval = client.evals().create(params);\n }\n}" ruby: "require \"openai\"\n\nopenai = OpenAI::Client.new(api_key: \"My API Key\")\n\neval_ = openai.evals.create(\n data_source_config: {item_schema: {foo: \"bar\"}, type: :custom},\n testing_criteria: [\n {\n input: [{content: \"content\", role: \"role\"}],\n labels: [\"string\"],\n model: \"model\",\n name: \"name\",\n passing_labels: [\"string\"],\n type: :label_model\n }\n ]\n)\n\nputs(eval_)" response: "{\n \"object\": \"eval\",\n \"id\": \"eval_67b7fa9a81a88190ab4aa417e397ea21\",\n \"data_source_config\": {\n \"type\": \"stored_completions\",\n \"metadata\": {\n \"usecase\": \"chatbot\"\n },\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\"\n },\n \"sample\": {\n \"type\": \"object\"\n }\n },\n \"required\": [\n \"item\",\n \"sample\"\n ]\n },\n \"testing_criteria\": [\n {\n \"name\": \"Example label grader\",\n \"type\": \"label_model\",\n \"model\": \"o3-mini\",\n \"input\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Classify the sentiment of the following statement as one of positive, neutral, or negative\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Statement: {{item.input}}\"\n }\n }\n ],\n \"passing_labels\": [\n \"positive\"\n ],\n \"labels\": [\n \"positive\",\n \"neutral\",\n \"negative\"\n ]\n }\n ],\n \"name\": \"Sentiment\",\n \"created_at\": 1740110490,\n \"metadata\": {\n \"description\": \"An eval for sentiment analysis\"\n }\n}\n" /evals/{eval_id}: get: operationId: getEval tags: - Evals summary: 'Get an evaluation by ID. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to retrieve. responses: '200': description: The evaluation content: application/json: schema: $ref: '#/components/schemas/Eval' x-oaiMeta: name: Get an eval group: evals path: get examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67abd54d9b0081909a86353f6fb9317a \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\neval = client.evals.retrieve(\n \"eval_id\",\n)\nprint(eval.id)" javascript: 'import OpenAI from "openai"; const openai = new OpenAI(); const evalObj = await openai.evals.retrieve("eval_67abd54d9b0081909a86353f6fb9317a"); console.log(evalObj); ' node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst _eval = await client.evals.retrieve('eval_id');\n\nconsole.log(_eval.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.EvalRetrieveParams;\nimport com.openai.models.evals.EvalRetrieveResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n EvalRetrieveResponse eval = client.evals().retrieve(\"eval_id\");\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") eval_ = openai.evals.retrieve("eval_id") puts(eval_)' response: "{\n \"object\": \"eval\",\n \"id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"data_source_config\": {\n \"type\": \"custom\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\",\n \"properties\": {\n \"input\": {\n \"type\": \"string\"\n },\n \"ground_truth\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"input\",\n \"ground_truth\"\n ]\n }\n },\n \"required\": [\n \"item\"\n ]\n }\n },\n \"testing_criteria\": [\n {\n \"name\": \"String check\",\n \"id\": \"String check-2eaf2d8d-d649-4335-8148-9535a7ca73c2\",\n \"type\": \"string_check\",\n \"input\": \"{{item.input}}\",\n \"reference\": \"{{item.ground_truth}}\",\n \"operation\": \"eq\"\n }\n ],\n \"name\": \"External Data Eval\",\n \"created_at\": 1739314509,\n \"metadata\": {},\n}\n" post: operationId: updateEval tags: - Evals summary: 'Update certain properties of an evaluation. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to update. requestBody: description: Request to update an evaluation required: true content: application/json: schema: type: object properties: name: type: string description: Rename the evaluation. metadata: $ref: '#/components/schemas/Metadata' responses: '200': description: The updated evaluation content: application/json: schema: $ref: '#/components/schemas/Eval' x-oaiMeta: name: Update an eval group: evals path: update examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67abd54d9b0081909a86353f6fb9317a \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"name\": \"Updated Eval\", \"metadata\": {\"description\": \"Updated description\"}}'\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\neval = client.evals.update(\n eval_id=\"eval_id\",\n)\nprint(eval.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst updatedEval = await openai.evals.update(\n \"eval_67abd54d9b0081909a86353f6fb9317a\",\n {\n name: \"Updated Eval\",\n metadata: { description: \"Updated description\" }\n }\n);\nconsole.log(updatedEval);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst _eval = await client.evals.update('eval_id');\n\nconsole.log(_eval.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.EvalUpdateParams;\nimport com.openai.models.evals.EvalUpdateResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n EvalUpdateResponse eval = client.evals().update(\"eval_id\");\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") eval_ = openai.evals.update("eval_id") puts(eval_)' response: "{\n \"object\": \"eval\",\n \"id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"data_source_config\": {\n \"type\": \"custom\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\",\n \"properties\": {\n \"input\": {\n \"type\": \"string\"\n },\n \"ground_truth\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"input\",\n \"ground_truth\"\n ]\n }\n },\n \"required\": [\n \"item\"\n ]\n }\n },\n \"testing_criteria\": [\n {\n \"name\": \"String check\",\n \"id\": \"String check-2eaf2d8d-d649-4335-8148-9535a7ca73c2\",\n \"type\": \"string_check\",\n \"input\": \"{{item.input}}\",\n \"reference\": \"{{item.ground_truth}}\",\n \"operation\": \"eq\"\n }\n ],\n \"name\": \"Updated Eval\",\n \"created_at\": 1739314509,\n \"metadata\": {\"description\": \"Updated description\"},\n}\n" delete: operationId: deleteEval tags: - Evals summary: 'Delete an evaluation. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to delete. responses: '200': description: Successfully deleted the evaluation. content: application/json: schema: type: object properties: object: type: string example: eval.deleted deleted: type: boolean example: true eval_id: type: string example: eval_abc123 required: - object - deleted - eval_id '404': description: Evaluation not found. content: application/json: schema: $ref: '#/components/schemas/Error' x-oaiMeta: name: Delete an eval group: evals examples: request: curl: "curl https://api.openai.com/v1/evals/eval_abc123 \\\n -X DELETE \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\neval = client.evals.delete(\n \"eval_id\",\n)\nprint(eval.eval_id)" javascript: 'import OpenAI from "openai"; const openai = new OpenAI(); const deleted = await openai.evals.delete("eval_abc123"); console.log(deleted); ' node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst _eval = await client.evals.delete('eval_id');\n\nconsole.log(_eval.eval_id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.EvalDeleteParams;\nimport com.openai.models.evals.EvalDeleteResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n EvalDeleteResponse eval = client.evals().delete(\"eval_id\");\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") eval_ = openai.evals.delete("eval_id") puts(eval_)' response: "{\n \"object\": \"eval.deleted\",\n \"deleted\": true,\n \"eval_id\": \"eval_abc123\"\n}\n" /evals/{eval_id}/runs: get: operationId: getEvalRuns tags: - Evals summary: 'Get a list of runs for an evaluation. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to retrieve runs for. - name: after in: query description: Identifier for the last run from the previous pagination request. required: false schema: type: string - name: limit in: query description: Number of runs to retrieve. required: false schema: type: integer default: 20 - name: order in: query description: Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for descending order. Defaults to `asc`. required: false schema: type: string enum: - asc - desc default: asc - name: status in: query description: Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed` | `canceled`. required: false schema: type: string enum: - queued - in_progress - completed - canceled - failed responses: '200': description: A list of runs for the evaluation content: application/json: schema: $ref: '#/components/schemas/EvalRunList' x-oaiMeta: name: Get eval runs group: evals path: get-runs examples: request: curl: "curl https://api.openai.com/v1/evals/egroup_67abd54d9b0081909a86353f6fb9317a/runs \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\npage = client.evals.runs.list(\n eval_id=\"eval_id\",\n)\npage = page.data[0]\nprint(page.id)" javascript: 'import OpenAI from "openai"; const openai = new OpenAI(); const runs = await openai.evals.runs.list("egroup_67abd54d9b0081909a86353f6fb9317a"); console.log(runs); ' node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const runListResponse of client.evals.runs.list('eval_id')) {\n console.log(runListResponse.id);\n}" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.RunListPage;\nimport com.openai.models.evals.runs.RunListParams;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n RunListPage page = client.evals().runs().list(\"eval_id\");\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") page = openai.evals.runs.list("eval_id") puts(page)' response: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67e0c7d31560819090d60c0780591042\",\n \"eval_id\": \"eval_67e0c726d560819083f19a957c4c640b\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67e0c726d560819083f19a957c4c640b\",\n \"status\": \"completed\",\n \"model\": \"o3-mini\",\n \"name\": \"bulk_with_negative_examples_o3-mini\",\n \"created_at\": 1742784467,\n \"result_counts\": {\n \"total\": 1,\n \"errored\": 0,\n \"failed\": 0,\n \"passed\": 1\n },\n \"per_model_usage\": [\n {\n \"model_name\": \"o3-mini\",\n \"invocation_count\": 1,\n \"prompt_tokens\": 563,\n \"completion_tokens\": 874,\n \"total_tokens\": 1437,\n \"cached_tokens\": 0\n }\n ],\n \"per_testing_criteria_results\": [\n {\n \"testing_criteria\": \"Push Notification Summary Grader-1808cd0b-eeec-4e0b-a519-337e79f4f5d1\",\n \"passed\": 1,\n \"failed\": 0\n }\n ],\n \"data_source\": {\n \"type\": \"completions\",\n \"source\": {\n \"type\": \"file_content\",\n \"content\": [\n {\n \"item\": {\n \"notifications\": \"\\n- New message from Sarah: \\\"Can you call me later?\\\"\\n- Your package has been delivered!\\n- Flash sale: 20% off electronics for the next 2 hours!\\n\"\n }\n }\n ]\n },\n \"input_messages\": {\n \"type\": \"template\",\n \"template\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"\\n\\n\\n\\nYou are a helpful assistant that takes in an array of push notifications and returns a collapsed summary of them.\\nThe push notification will be provided as follows:\\n\\n...notificationlist...\\n\\n\\nYou should return just the summary and nothing else.\\n\\n\\nYou should return a summary that is concise and snappy.\\n\\n\\nHere is an example of a good summary:\\n\\n- Traffic alert: Accident reported on Main Street.- Package out for delivery: Expected by 5 PM.- New friend suggestion: Connect with Emma.\\n\\n\\nTraffic alert, package expected by 5pm, suggestion for new friend (Emily).\\n\\n\\n\\nHere is an example of a bad summary:\\n\\n- Traffic alert: Accident reported on Main Street.- Package out for delivery: Expected by 5 PM.- New friend suggestion: Connect with Emma.\\n\\n\\nTraffic alert reported on main street. You have a package that will arrive by 5pm, Emily is a new friend suggested for you.\\n\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"{{item.notifications}}\"\n }\n }\n ]\n },\n \"model\": \"o3-mini\",\n \"sampling_params\": null\n },\n \"error\": null,\n \"metadata\": {}\n }\n ],\n \"first_id\": \"evalrun_67e0c7d31560819090d60c0780591042\",\n \"last_id\": \"evalrun_67e0c7d31560819090d60c0780591042\",\n \"has_more\": true\n}\n" post: operationId: createEvalRun tags: - Evals summary: 'Kicks off a new run for a given evaluation, specifying the data source, and what model configuration to use to test. The datasource will be validated against the schema specified in the config of the evaluation. ' parameters: - in: path name: eval_id required: true schema: type: string description: The ID of the evaluation to create a run for. requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/CreateEvalRunRequest' responses: '201': description: Successfully created a run for the evaluation content: application/json: schema: $ref: '#/components/schemas/EvalRun' '400': description: Bad request (for example, missing eval object) content: application/json: schema: $ref: '#/components/schemas/Error' x-oaiMeta: name: Create eval run group: evals examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67e579652b548190aaa83ada4b125f47/runs \\\n -X POST \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"name\":\"gpt-4o-mini\",\"data_source\":{\"type\":\"completions\",\"input_messages\":{\"type\":\"template\",\"template\":[{\"role\":\"developer\",\"content\":\"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"} , {\"role\":\"user\",\"content\":\"{{item.input}}\"}]} ,\"sampling_params\":{\"temperature\":1,\"max_completions_tokens\":2048,\"top_p\":1,\"seed\":42},\"model\":\"gpt-4o-mini\",\"source\":{\"type\":\"file_content\",\"content\":[{\"item\":{\"input\":\"Tech Company Launches Advanced Artificial Intelligence Platform\",\"ground_truth\":\"Technology\"}}]}}'\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\nrun = client.evals.runs.create(\n eval_id=\"eval_id\",\n data_source={\n \"source\": {\n \"content\": [{\n \"item\": {\n \"foo\": \"bar\"\n }\n }],\n \"type\": \"file_content\",\n },\n \"type\": \"jsonl\",\n },\n)\nprint(run.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst run = await openai.evals.runs.create(\n \"eval_67e579652b548190aaa83ada4b125f47\",\n {\n name: \"gpt-4o-mini\",\n data_source: {\n type: \"completions\",\n input_messages: {\n type: \"template\",\n template: [\n {\n role: \"developer\",\n content: \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"\n },\n {\n role: \"user\",\n content: \"{{item.input}}\"\n }\n ]\n },\n sampling_params: {\n temperature: 1,\n max_completions_tokens: 2048,\n top_p: 1,\n seed: 42\n },\n model: \"gpt-4o-mini\",\n source: {\n type: \"file_content\",\n content: [\n {\n item: {\n input: \"Tech Company Launches Advanced Artificial Intelligence Platform\",\n ground_truth: \"Technology\"\n }\n }\n ]\n }\n }\n }\n);\nconsole.log(run);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst run = await client.evals.runs.create('eval_id', {\n data_source: {\n source: { content: [{ item: { foo: 'bar' } }], type: 'file_content' },\n type: 'jsonl',\n },\n});\n\nconsole.log(run.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.core.JsonValue;\nimport com.openai.models.evals.runs.CreateEvalJsonlRunDataSource;\nimport com.openai.models.evals.runs.RunCreateParams;\nimport com.openai.models.evals.runs.RunCreateResponse;\nimport java.util.List;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n RunCreateParams params = RunCreateParams.builder()\n .evalId(\"eval_id\")\n .dataSource(CreateEvalJsonlRunDataSource.builder()\n .fileContentSource(List.of(CreateEvalJsonlRunDataSource.Source.FileContent.Content.builder()\n .item(CreateEvalJsonlRunDataSource.Source.FileContent.Content.Item.builder()\n .putAdditionalProperty(\"foo\", JsonValue.from(\"bar\"))\n .build())\n .build()))\n .build())\n .build();\n RunCreateResponse run = client.evals().runs().create(params);\n }\n}" ruby: "require \"openai\"\n\nopenai = OpenAI::Client.new(api_key: \"My API Key\")\n\nrun = openai.evals.runs.create(\n \"eval_id\",\n data_source: {source: {content: [{item: {foo: \"bar\"}}], type: :file_content}, type: :jsonl}\n)\n\nputs(run)" response: "{\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67e57965b480819094274e3a32235e4c\",\n \"eval_id\": \"eval_67e579652b548190aaa83ada4b125f47\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67e579652b548190aaa83ada4b125f47&run_id=evalrun_67e57965b480819094274e3a32235e4c\",\n \"status\": \"queued\",\n \"model\": \"gpt-4o-mini\",\n \"name\": \"gpt-4o-mini\",\n \"created_at\": 1743092069,\n \"result_counts\": {\n \"total\": 0,\n \"errored\": 0,\n \"failed\": 0,\n \"passed\": 0\n },\n \"per_model_usage\": null,\n \"per_testing_criteria_results\": null,\n \"data_source\": {\n \"type\": \"completions\",\n \"source\": {\n \"type\": \"file_content\",\n \"content\": [\n {\n \"item\": {\n \"input\": \"Tech Company Launches Advanced Artificial Intelligence Platform\",\n \"ground_truth\": \"Technology\"\n }\n }\n ]\n },\n \"input_messages\": {\n \"type\": \"template\",\n \"template\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"{{item.input}}\"\n }\n }\n ]\n },\n \"model\": \"gpt-4o-mini\",\n \"sampling_params\": {\n \"seed\": 42,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n \"max_completions_tokens\": 2048\n }\n },\n \"error\": null,\n \"metadata\": {}\n}\n" /evals/{eval_id}/runs/{run_id}: get: operationId: getEvalRun tags: - Evals summary: 'Get an evaluation run by ID. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to retrieve runs for. - name: run_id in: path required: true schema: type: string description: The ID of the run to retrieve. responses: '200': description: The evaluation run content: application/json: schema: $ref: '#/components/schemas/EvalRun' x-oaiMeta: name: Get an eval run group: evals path: get examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67abd54d9b0081909a86353f6fb9317a/runs/evalrun_67abd54d60ec8190832b46859da808f7 \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\nrun = client.evals.runs.retrieve(\n run_id=\"run_id\",\n eval_id=\"eval_id\",\n)\nprint(run.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst run = await openai.evals.runs.retrieve(\n \"evalrun_67abd54d60ec8190832b46859da808f7\",\n { eval_id: \"eval_67abd54d9b0081909a86353f6fb9317a\" }\n);\nconsole.log(run);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst run = await client.evals.runs.retrieve('run_id', { eval_id: 'eval_id' });\n\nconsole.log(run.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.RunRetrieveParams;\nimport com.openai.models.evals.runs.RunRetrieveResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n RunRetrieveParams params = RunRetrieveParams.builder()\n .evalId(\"eval_id\")\n .runId(\"run_id\")\n .build();\n RunRetrieveResponse run = client.evals().runs().retrieve(params);\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") run = openai.evals.runs.retrieve("run_id", eval_id: "eval_id") puts(run)' response: "{\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67abd54d9b0081909a86353f6fb9317a?run_id=evalrun_67abd54d60ec8190832b46859da808f7\",\n \"status\": \"queued\",\n \"model\": \"gpt-4o-mini\",\n \"name\": \"gpt-4o-mini\",\n \"created_at\": 1743092069,\n \"result_counts\": {\n \"total\": 0,\n \"errored\": 0,\n \"failed\": 0,\n \"passed\": 0\n },\n \"per_model_usage\": null,\n \"per_testing_criteria_results\": null,\n \"data_source\": {\n \"type\": \"completions\",\n \"source\": {\n \"type\": \"file_content\",\n \"content\": [\n {\n \"item\": {\n \"input\": \"Tech Company Launches Advanced Artificial Intelligence Platform\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Central Bank Increases Interest Rates Amid Inflation Concerns\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Summit Addresses Climate Change Strategies\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Major Retailer Reports Record-Breaking Holiday Sales\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"National Team Qualifies for World Championship Finals\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Manufacturer Announces Merger with Competitor\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Breakthrough in Renewable Energy Technology Unveiled\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"World Leaders Sign Historic Climate Agreement\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Professional Athlete Sets New Record in Championship Event\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Financial Institutions Adapt to New Regulatory Requirements\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Tech Conference Showcases Advances in Artificial Intelligence\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Markets Respond to Oil Price Fluctuations\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Cooperation Strengthened Through New Treaty\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Sports League Announces Revised Schedule for Upcoming Season\",\n \"ground_truth\": \"Sports\"\n }\n }\n ]\n },\n \"input_messages\": {\n \"type\": \"template\",\n \"template\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"{{item.input}}\"\n }\n }\n ]\n },\n \"model\": \"gpt-4o-mini\",\n \"sampling_params\": {\n \"seed\": 42,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n \"max_completions_tokens\": 2048\n }\n },\n \"error\": null,\n \"metadata\": {}\n}\n" post: operationId: cancelEvalRun tags: - Evals summary: 'Cancel an ongoing evaluation run. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation whose run you want to cancel. - name: run_id in: path required: true schema: type: string description: The ID of the run to cancel. responses: '200': description: The canceled eval run object content: application/json: schema: $ref: '#/components/schemas/EvalRun' x-oaiMeta: name: Cancel eval run group: evals path: post examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67abd54d9b0081909a86353f6fb9317a/runs/evalrun_67abd54d60ec8190832b46859da808f7/cancel \\\n -X POST \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\nresponse = client.evals.runs.cancel(\n run_id=\"run_id\",\n eval_id=\"eval_id\",\n)\nprint(response.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst canceledRun = await openai.evals.runs.cancel(\n \"evalrun_67abd54d60ec8190832b46859da808f7\",\n { eval_id: \"eval_67abd54d9b0081909a86353f6fb9317a\" }\n);\nconsole.log(canceledRun);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst response = await client.evals.runs.cancel('run_id', { eval_id: 'eval_id' });\n\nconsole.log(response.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.RunCancelParams;\nimport com.openai.models.evals.runs.RunCancelResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n RunCancelParams params = RunCancelParams.builder()\n .evalId(\"eval_id\")\n .runId(\"run_id\")\n .build();\n RunCancelResponse response = client.evals().runs().cancel(params);\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") response = openai.evals.runs.cancel("run_id", eval_id: "eval_id") puts(response)' response: "{\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67abd54d9b0081909a86353f6fb9317a?run_id=evalrun_67abd54d60ec8190832b46859da808f7\",\n \"status\": \"canceled\",\n \"model\": \"gpt-4o-mini\",\n \"name\": \"gpt-4o-mini\",\n \"created_at\": 1743092069,\n \"result_counts\": {\n \"total\": 0,\n \"errored\": 0,\n \"failed\": 0,\n \"passed\": 0\n },\n \"per_model_usage\": null,\n \"per_testing_criteria_results\": null,\n \"data_source\": {\n \"type\": \"completions\",\n \"source\": {\n \"type\": \"file_content\",\n \"content\": [\n {\n \"item\": {\n \"input\": \"Tech Company Launches Advanced Artificial Intelligence Platform\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Central Bank Increases Interest Rates Amid Inflation Concerns\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Summit Addresses Climate Change Strategies\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Major Retailer Reports Record-Breaking Holiday Sales\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"National Team Qualifies for World Championship Finals\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Manufacturer Announces Merger with Competitor\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Breakthrough in Renewable Energy Technology Unveiled\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"World Leaders Sign Historic Climate Agreement\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Professional Athlete Sets New Record in Championship Event\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Financial Institutions Adapt to New Regulatory Requirements\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Tech Conference Showcases Advances in Artificial Intelligence\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Markets Respond to Oil Price Fluctuations\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Cooperation Strengthened Through New Treaty\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Sports League Announces Revised Schedule for Upcoming Season\",\n \"ground_truth\": \"Sports\"\n }\n }\n ]\n },\n \"input_messages\": {\n \"type\": \"template\",\n \"template\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"{{item.input}}\"\n }\n }\n ]\n },\n \"model\": \"gpt-4o-mini\",\n \"sampling_params\": {\n \"seed\": 42,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n \"max_completions_tokens\": 2048\n }\n },\n \"error\": null,\n \"metadata\": {}\n}\n" delete: operationId: deleteEvalRun tags: - Evals summary: 'Delete an eval run. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to delete the run from. - name: run_id in: path required: true schema: type: string description: The ID of the run to delete. responses: '200': description: Successfully deleted the eval run content: application/json: schema: type: object properties: object: type: string example: eval.run.deleted deleted: type: boolean example: true run_id: type: string example: evalrun_677469f564d48190807532a852da3afb '404': description: Run not found content: application/json: schema: $ref: '#/components/schemas/Error' x-oaiMeta: name: Delete eval run group: evals path: delete examples: request: curl: "curl https://api.openai.com/v1/evals/eval_123abc/runs/evalrun_abc456 \\\n -X DELETE \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\nrun = client.evals.runs.delete(\n run_id=\"run_id\",\n eval_id=\"eval_id\",\n)\nprint(run.run_id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst deleted = await openai.evals.runs.delete(\n \"eval_123abc\",\n \"evalrun_abc456\"\n);\nconsole.log(deleted);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst run = await client.evals.runs.delete('run_id', { eval_id: 'eval_id' });\n\nconsole.log(run.run_id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.RunDeleteParams;\nimport com.openai.models.evals.runs.RunDeleteResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n RunDeleteParams params = RunDeleteParams.builder()\n .evalId(\"eval_id\")\n .runId(\"run_id\")\n .build();\n RunDeleteResponse run = client.evals().runs().delete(params);\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") run = openai.evals.runs.delete("run_id", eval_id: "eval_id") puts(run)' response: "{\n \"object\": \"eval.run.deleted\",\n \"deleted\": true,\n \"run_id\": \"evalrun_abc456\"\n}\n" /evals/{eval_id}/runs/{run_id}/output_items: get: operationId: getEvalRunOutputItems tags: - Evals summary: 'Get a list of output items for an evaluation run. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to retrieve runs for. - name: run_id in: path required: true schema: type: string description: The ID of the run to retrieve output items for. - name: after in: query description: Identifier for the last output item from the previous pagination request. required: false schema: type: string - name: limit in: query description: Number of output items to retrieve. required: false schema: type: integer default: 20 - name: status in: query description: 'Filter output items by status. Use `failed` to filter by failed output items or `pass` to filter by passed output items. ' required: false schema: type: string enum: - fail - pass - name: order in: query description: Sort order for output items by timestamp. Use `asc` for ascending order or `desc` for descending order. Defaults to `asc`. required: false schema: type: string enum: - asc - desc default: asc responses: '200': description: A list of output items for the evaluation run content: application/json: schema: $ref: '#/components/schemas/EvalRunOutputItemList' x-oaiMeta: name: Get eval run output items group: evals path: get examples: request: curl: "curl https://api.openai.com/v1/evals/egroup_67abd54d9b0081909a86353f6fb9317a/runs/erun_67abd54d60ec8190832b46859da808f7/output_items \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\npage = client.evals.runs.output_items.list(\n run_id=\"run_id\",\n eval_id=\"eval_id\",\n)\npage = page.data[0]\nprint(page.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst outputItems = await openai.evals.runs.outputItems.list(\n \"egroup_67abd54d9b0081909a86353f6fb9317a\",\n \"erun_67abd54d60ec8190832b46859da808f7\"\n);\nconsole.log(outputItems);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const outputItemListResponse of client.evals.runs.outputItems.list('run_id', {\n eval_id: 'eval_id',\n})) {\n console.log(outputItemListResponse.id);\n}" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.outputitems.OutputItemListPage;\nimport com.openai.models.evals.runs.outputitems.OutputItemListParams;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n OutputItemListParams params = OutputItemListParams.builder()\n .evalId(\"eval_id\")\n .runId(\"run_id\")\n .build();\n OutputItemListPage page = client.evals().runs().outputItems().list(params);\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") page = openai.evals.runs.output_items.list("run_id", eval_id: "eval_id") puts(page)' response: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"eval.run.output_item\",\n \"id\": \"outputitem_67e5796c28e081909917bf79f6e6214d\",\n \"created_at\": 1743092076,\n \"run_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"status\": \"pass\",\n \"datasource_item_id\": 5,\n \"datasource_item\": {\n \"input\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"ground_truth\": \"Markets\"\n },\n \"results\": [\n {\n \"name\": \"String check-a2486074-d803-4445-b431-ad2262e85d47\",\n \"sample\": null,\n \"passed\": true,\n \"score\": 1.0\n }\n ],\n \"sample\": {\n \"input\": [\n {\n \"role\": \"developer\",\n \"content\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n },\n {\n \"role\": \"user\",\n \"content\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n }\n ],\n \"output\": [\n {\n \"role\": \"assistant\",\n \"content\": \"Markets\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n }\n ],\n \"finish_reason\": \"stop\",\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"usage\": {\n \"total_tokens\": 325,\n \"completion_tokens\": 2,\n \"prompt_tokens\": 323,\n \"cached_tokens\": 0\n },\n \"error\": null,\n \"temperature\": 1.0,\n \"max_completion_tokens\": 2048,\n \"top_p\": 1.0,\n \"seed\": 42\n }\n }\n ],\n \"first_id\": \"outputitem_67e5796c28e081909917bf79f6e6214d\",\n \"last_id\": \"outputitem_67e5796c28e081909917bf79f6e6214d\",\n \"has_more\": true\n}\n" /evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}: get: operationId: getEvalRunOutputItem tags: - Evals summary: 'Get an evaluation run output item by ID. ' parameters: - name: eval_id in: path required: true schema: type: string description: The ID of the evaluation to retrieve runs for. - name: run_id in: path required: true schema: type: string description: The ID of the run to retrieve. - name: output_item_id in: path required: true schema: type: string description: The ID of the output item to retrieve. responses: '200': description: The evaluation run output item content: application/json: schema: $ref: '#/components/schemas/EvalRunOutputItem' x-oaiMeta: name: Get an output item of an eval run group: evals path: get examples: request: curl: "curl https://api.openai.com/v1/evals/eval_67abd54d9b0081909a86353f6fb9317a/runs/evalrun_67abd54d60ec8190832b46859da808f7/output_items/outputitem_67abd55eb6548190bb580745d5644a33 \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -H \"Content-Type: application/json\"\n" python: "import os\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=os.environ.get(\"OPENAI_API_KEY\"), # This is the default and can be omitted\n)\noutput_item = client.evals.runs.output_items.retrieve(\n output_item_id=\"output_item_id\",\n eval_id=\"eval_id\",\n run_id=\"run_id\",\n)\nprint(output_item.id)" javascript: "import OpenAI from \"openai\";\n\nconst openai = new OpenAI();\n\nconst outputItem = await openai.evals.runs.outputItems.retrieve(\n \"outputitem_67abd55eb6548190bb580745d5644a33\",\n {\n eval_id: \"eval_67abd54d9b0081909a86353f6fb9317a\",\n run_id: \"evalrun_67abd54d60ec8190832b46859da808f7\",\n }\n);\nconsole.log(outputItem);\n" node.js: "import OpenAI from 'openai';\n\nconst client = new OpenAI({\n apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted\n});\n\nconst outputItem = await client.evals.runs.outputItems.retrieve('output_item_id', {\n eval_id: 'eval_id',\n run_id: 'run_id',\n});\n\nconsole.log(outputItem.id);" java: "package com.openai.example;\n\nimport com.openai.client.OpenAIClient;\nimport com.openai.client.okhttp.OpenAIOkHttpClient;\nimport com.openai.models.evals.runs.outputitems.OutputItemRetrieveParams;\nimport com.openai.models.evals.runs.outputitems.OutputItemRetrieveResponse;\n\npublic final class Main {\n private Main() {}\n\n public static void main(String[] args) {\n OpenAIClient client = OpenAIOkHttpClient.fromEnv();\n\n OutputItemRetrieveParams params = OutputItemRetrieveParams.builder()\n .evalId(\"eval_id\")\n .runId(\"run_id\")\n .outputItemId(\"output_item_id\")\n .build();\n OutputItemRetrieveResponse outputItem = client.evals().runs().outputItems().retrieve(params);\n }\n}" ruby: 'require "openai" openai = OpenAI::Client.new(api_key: "My API Key") output_item = openai.evals.runs.output_items.retrieve("output_item_id", eval_id: "eval_id", run_id: "run_id") puts(output_item)' response: "{\n \"object\": \"eval.run.output_item\",\n \"id\": \"outputitem_67e5796c28e081909917bf79f6e6214d\",\n \"created_at\": 1743092076,\n \"run_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"status\": \"pass\",\n \"datasource_item_id\": 5,\n \"datasource_item\": {\n \"input\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"ground_truth\": \"Markets\"\n },\n \"results\": [\n {\n \"name\": \"String check-a2486074-d803-4445-b431-ad2262e85d47\",\n \"sample\": null,\n \"passed\": true,\n \"score\": 1.0\n }\n ],\n \"sample\": {\n \"input\": [\n {\n \"role\": \"developer\",\n \"content\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n },\n {\n \"role\": \"user\",\n \"content\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n }\n ],\n \"output\": [\n {\n \"role\": \"assistant\",\n \"content\": \"Markets\",\n \"tool_call_id\": null,\n \"tool_calls\": null,\n \"function_call\": null\n }\n ],\n \"finish_reason\": \"stop\",\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"usage\": {\n \"total_tokens\": 325,\n \"completion_tokens\": 2,\n \"prompt_tokens\": 323,\n \"cached_tokens\": 0\n },\n \"error\": null,\n \"temperature\": 1.0,\n \"max_completion_tokens\": 2048,\n \"top_p\": 1.0,\n \"seed\": 42\n }\n}\n" components: schemas: EvalRunOutputItemList: type: object title: EvalRunOutputItemList description: 'An object representing a list of output items for an evaluation run. ' properties: object: type: string enum: - list default: list description: 'The type of this object. It is always set to "list". ' x-stainless-const: true data: type: array description: 'An array of eval run output item objects. ' items: $ref: '#/components/schemas/EvalRunOutputItem' first_id: type: string description: The identifier of the first eval run output item in the data array. last_id: type: string description: The identifier of the last eval run output item in the data array. has_more: type: boolean description: Indicates whether there are more eval run output items available. required: - object - data - first_id - last_id - has_more x-oaiMeta: name: The eval run output item list object group: evals example: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"eval.run.output_item\",\n \"id\": \"outputitem_67abd55eb6548190bb580745d5644a33\",\n \"run_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"created_at\": 1739314509,\n \"status\": \"pass\",\n \"datasource_item_id\": 137,\n \"datasource_item\": {\n \"teacher\": \"To grade essays, I only check for style, content, and grammar.\",\n \"student\": \"I am a student who is trying to write the best essay.\"\n },\n \"results\": [\n {\n \"name\": \"String Check Grader\",\n \"type\": \"string-check-grader\",\n \"score\": 1.0,\n \"passed\": true,\n }\n ],\n \"sample\": {\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an evaluator bot...\"\n },\n {\n \"role\": \"user\",\n \"content\": \"You are assessing...\"\n }\n ],\n \"output\": [\n {\n \"role\": \"assistant\",\n \"content\": \"The rubric is not clear nor concise.\"\n }\n ],\n \"finish_reason\": \"stop\",\n \"model\": \"gpt-4o-2024-08-06\",\n \"usage\": {\n \"total_tokens\": 521,\n \"completion_tokens\": 2,\n \"prompt_tokens\": 519,\n \"cached_tokens\": 0\n },\n \"error\": null,\n \"temperature\": 1.0,\n \"max_completion_tokens\": 2048,\n \"top_p\": 1.0,\n \"seed\": 42\n }\n },\n ],\n \"first_id\": \"outputitem_67abd55eb6548190bb580745d5644a33\",\n \"last_id\": \"outputitem_67abd55eb6548190bb580745d5644a33\",\n \"has_more\": false\n}\n" EvalRunOutputItemResult: type: object title: EvalRunOutputItemResult description: 'A single grader result for an evaluation run output item. ' properties: name: type: string description: The name of the grader. type: type: string description: The grader type (for example, "string-check-grader"). score: type: number description: The numeric score produced by the grader. passed: type: boolean description: Whether the grader considered the output a pass. sample: anyOf: - type: object additionalProperties: true - type: 'null' description: Optional sample or intermediate data produced by the grader. additionalProperties: true required: - name - score - passed EvalItemInputImage: title: Input image description: An image input block used within EvalItem content arrays. type: object properties: type: type: string description: 'The type of the image input. Always `input_image`. ' enum: - input_image x-stainless-const: true image_url: type: string format: uri description: 'The URL of the image input. ' detail: type: string description: 'The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. ' required: - type - image_url WebSearchTool: type: object title: Web search description: 'Search the Internet for sources related to the prompt. Learn more about the [web search tool](/docs/guides/tools-web-search). ' properties: type: type: string enum: - web_search - web_search_2025_08_26 description: The type of the web search tool. One of `web_search` or `web_search_2025_08_26`. default: web_search filters: anyOf: - type: object description: 'Filters for the search. ' properties: allowed_domains: anyOf: - type: array title: Allowed domains for the search. description: 'Allowed domains for the search. If not provided, all domains are allowed. Subdomains of the provided domains are allowed as well. Example: `["pubmed.ncbi.nlm.nih.gov"]` ' items: type: string description: Allowed domain for the search. default: [] - type: 'null' - type: 'null' user_location: $ref: '#/components/schemas/WebSearchApproximateLocation' search_context_size: type: string enum: - low - medium - high default: medium description: High level guidance for the amount of context window space to use for the search. One of `low`, `medium`, or `high`. `medium` is the default. required: - type InlineSkillParam: properties: type: type: string enum: - inline description: Defines an inline skill for this request. default: inline x-stainless-const: true name: type: string description: The name of the skill. description: type: string description: The description of the skill. source: $ref: '#/components/schemas/InlineSkillSourceParam' description: Inline skill payload type: object required: - type - name - description - source ResponseFormatJsonSchema: type: object title: JSON schema description: 'JSON Schema response format. Used to generate structured JSON responses. Learn more about [Structured Outputs](/docs/guides/structured-outputs). ' properties: type: type: string description: The type of response format being defined. Always `json_schema`. enum: - json_schema x-stainless-const: true json_schema: type: object title: JSON schema description: 'Structured Outputs configuration options, including a JSON Schema. ' properties: description: type: string description: 'A description of what the response format is for, used by the model to determine how to respond in the format. ' name: type: string description: 'The name of the response format. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. ' schema: $ref: '#/components/schemas/ResponseFormatJsonSchemaSchema' strict: anyOf: - type: boolean default: false description: 'Whether to enable strict schema adherence when generating the output. If set to true, the model will always follow the exact schema defined in the `schema` field. Only a subset of JSON Schema is supported when `strict` is `true`. To learn more, read the [Structured Outputs guide](/docs/guides/structured-outputs). ' - type: 'null' required: - name required: - type - json_schema ImageGenActionEnum: type: string enum: - generate - edit - auto EvalGraderScoreModel: type: object title: ScoreModelGrader allOf: - $ref: '#/components/schemas/GraderScoreModel' - type: object properties: pass_threshold: type: number description: The threshold for the score. FileInputDetail: type: string enum: - low - high EvalList: type: object title: EvalList description: 'An object representing a list of evals. ' properties: object: type: string enum: - list default: list description: 'The type of this object. It is always set to "list". ' x-stainless-const: true data: type: array description: 'An array of eval objects. ' items: $ref: '#/components/schemas/Eval' first_id: type: string description: The identifier of the first eval in the data array. last_id: type: string description: The identifier of the last eval in the data array. has_more: type: boolean description: Indicates whether there are more evals available. required: - object - data - first_id - last_id - has_more x-oaiMeta: name: The eval list object group: evals example: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"eval\",\n \"id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"data_source_config\": {\n \"type\": \"custom\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\",\n \"properties\": {\n \"input\": {\n \"type\": \"string\"\n },\n \"ground_truth\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"input\",\n \"ground_truth\"\n ]\n }\n },\n \"required\": [\n \"item\"\n ]\n }\n },\n \"testing_criteria\": [\n {\n \"name\": \"String check\",\n \"id\": \"String check-2eaf2d8d-d649-4335-8148-9535a7ca73c2\",\n \"type\": \"string_check\",\n \"input\": \"{{item.input}}\",\n \"reference\": \"{{item.ground_truth}}\",\n \"operation\": \"eq\"\n }\n ],\n \"name\": \"External Data Eval\",\n \"created_at\": 1739314509,\n \"metadata\": {},\n }\n ],\n \"first_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"last_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"has_more\": true\n}\n" SearchContentType: type: string enum: - text - image Eval: type: object title: Eval description: "An Eval object with a data source config and testing criteria.\nAn Eval represents a task to be done for your LLM integration.\nLike:\n - Improve the quality of my chatbot\n - See how well my chatbot handles customer support\n - Check if o4-mini is better at my usecase than gpt-4o\n" properties: object: type: string enum: - eval default: eval description: The object type. x-stainless-const: true id: type: string description: Unique identifier for the evaluation. name: type: string description: The name of the evaluation. example: Chatbot effectiveness Evaluation data_source_config: type: object description: Configuration of data sources used in runs of the evaluation. oneOf: - $ref: '#/components/schemas/EvalCustomDataSourceConfig' - $ref: '#/components/schemas/EvalLogsDataSourceConfig' - $ref: '#/components/schemas/EvalStoredCompletionsDataSourceConfig' testing_criteria: default: eval description: A list of testing criteria. type: array items: oneOf: - $ref: '#/components/schemas/EvalGraderLabelModel' - $ref: '#/components/schemas/EvalGraderStringCheck' - $ref: '#/components/schemas/EvalGraderTextSimilarity' - $ref: '#/components/schemas/EvalGraderPython' - $ref: '#/components/schemas/EvalGraderScoreModel' created_at: type: integer format: unixtime description: The Unix timestamp (in seconds) for when the eval was created. metadata: $ref: '#/components/schemas/Metadata' required: - id - data_source_config - object - testing_criteria - name - created_at - metadata x-oaiMeta: name: The eval object group: evals example: "{\n \"object\": \"eval\",\n \"id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"data_source_config\": {\n \"type\": \"custom\",\n \"item_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"label\": {\"type\": \"string\"},\n },\n \"required\": [\"label\"]\n },\n \"include_sample_schema\": true\n },\n \"testing_criteria\": [\n {\n \"name\": \"My string check grader\",\n \"type\": \"string_check\",\n \"input\": \"{{sample.output_text}}\",\n \"reference\": \"{{item.label}}\",\n \"operation\": \"eq\",\n }\n ],\n \"name\": \"External Data Eval\",\n \"created_at\": 1739314509,\n \"metadata\": {\n \"test\": \"synthetics\",\n }\n}\n" EvalRun: type: object title: EvalRun description: 'A schema representing an evaluation run. ' properties: object: type: string enum: - eval.run default: eval.run description: The type of the object. Always "eval.run". x-stainless-const: true id: type: string description: Unique identifier for the evaluation run. eval_id: type: string description: The identifier of the associated evaluation. status: type: string description: The status of the evaluation run. model: type: string description: The model that is evaluated, if applicable. name: type: string description: The name of the evaluation run. created_at: type: integer format: unixtime description: Unix timestamp (in seconds) when the evaluation run was created. report_url: type: string format: uri description: The URL to the rendered evaluation run report on the UI dashboard. result_counts: type: object description: Counters summarizing the outcomes of the evaluation run. properties: total: type: integer description: Total number of executed output items. errored: type: integer description: Number of output items that resulted in an error. failed: type: integer description: Number of output items that failed to pass the evaluation. passed: type: integer description: Number of output items that passed the evaluation. required: - total - errored - failed - passed per_model_usage: type: array description: Usage statistics for each model during the evaluation run. items: type: object properties: model_name: type: string description: The name of the model. invocation_count: type: integer description: The number of invocations. prompt_tokens: type: integer description: The number of prompt tokens used. completion_tokens: type: integer description: The number of completion tokens generated. total_tokens: type: integer description: The total number of tokens used. cached_tokens: type: integer description: The number of tokens retrieved from cache. required: - model_name - invocation_count - prompt_tokens - completion_tokens - total_tokens - cached_tokens per_testing_criteria_results: type: array description: Results per testing criteria applied during the evaluation run. items: type: object properties: testing_criteria: type: string description: A description of the testing criteria. passed: type: integer description: Number of tests passed for this criteria. failed: type: integer description: Number of tests failed for this criteria. required: - testing_criteria - passed - failed data_source: type: object description: Information about the run's data source. oneOf: - $ref: '#/components/schemas/CreateEvalJsonlRunDataSource' - $ref: '#/components/schemas/CreateEvalCompletionsRunDataSource' - $ref: '#/components/schemas/CreateEvalResponsesRunDataSource' metadata: $ref: '#/components/schemas/Metadata' error: $ref: '#/components/schemas/EvalApiError' required: - object - id - eval_id - status - model - name - created_at - report_url - result_counts - per_model_usage - per_testing_criteria_results - data_source - metadata - error x-oaiMeta: name: The eval run object group: evals example: "{\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67e57965b480819094274e3a32235e4c\",\n \"eval_id\": \"eval_67e579652b548190aaa83ada4b125f47\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67e579652b548190aaa83ada4b125f47?run_id=evalrun_67e57965b480819094274e3a32235e4c\",\n \"status\": \"queued\",\n \"model\": \"gpt-4o-mini\",\n \"name\": \"gpt-4o-mini\",\n \"created_at\": 1743092069,\n \"result_counts\": {\n \"total\": 0,\n \"errored\": 0,\n \"failed\": 0,\n \"passed\": 0\n },\n \"per_model_usage\": null,\n \"per_testing_criteria_results\": null,\n \"data_source\": {\n \"type\": \"completions\",\n \"source\": {\n \"type\": \"file_content\",\n \"content\": [\n {\n \"item\": {\n \"input\": \"Tech Company Launches Advanced Artificial Intelligence Platform\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Central Bank Increases Interest Rates Amid Inflation Concerns\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Summit Addresses Climate Change Strategies\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Major Retailer Reports Record-Breaking Holiday Sales\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"National Team Qualifies for World Championship Finals\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Stock Markets Rally After Positive Economic Data Released\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Manufacturer Announces Merger with Competitor\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Breakthrough in Renewable Energy Technology Unveiled\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"World Leaders Sign Historic Climate Agreement\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Professional Athlete Sets New Record in Championship Event\",\n \"ground_truth\": \"Sports\"\n }\n },\n {\n \"item\": {\n \"input\": \"Financial Institutions Adapt to New Regulatory Requirements\",\n \"ground_truth\": \"Business\"\n }\n },\n {\n \"item\": {\n \"input\": \"Tech Conference Showcases Advances in Artificial Intelligence\",\n \"ground_truth\": \"Technology\"\n }\n },\n {\n \"item\": {\n \"input\": \"Global Markets Respond to Oil Price Fluctuations\",\n \"ground_truth\": \"Markets\"\n }\n },\n {\n \"item\": {\n \"input\": \"International Cooperation Strengthened Through New Treaty\",\n \"ground_truth\": \"World\"\n }\n },\n {\n \"item\": {\n \"input\": \"Sports League Announces Revised Schedule for Upcoming Season\",\n \"ground_truth\": \"Sports\"\n }\n }\n ]\n },\n \"input_messages\": {\n \"type\": \"template\",\n \"template\": [\n {\n \"type\": \"message\",\n \"role\": \"developer\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Categorize a given news headline into one of the following topics: Technology, Markets, World, Business, or Sports.\\n\\n# Steps\\n\\n1. Analyze the content of the news headline to understand its primary focus.\\n2. Extract the subject matter, identifying any key indicators or keywords.\\n3. Use the identified indicators to determine the most suitable category out of the five options: Technology, Markets, World, Business, or Sports.\\n4. Ensure only one category is selected per headline.\\n\\n# Output Format\\n\\nRespond with the chosen category as a single word. For instance: \\\"Technology\\\", \\\"Markets\\\", \\\"World\\\", \\\"Business\\\", or \\\"Sports\\\".\\n\\n# Examples\\n\\n**Input**: \\\"Apple Unveils New iPhone Model, Featuring Advanced AI Features\\\" \\n**Output**: \\\"Technology\\\"\\n\\n**Input**: \\\"Global Stocks Mixed as Investors Await Central Bank Decisions\\\" \\n**Output**: \\\"Markets\\\"\\n\\n**Input**: \\\"War in Ukraine: Latest Updates on Negotiation Status\\\" \\n**Output**: \\\"World\\\"\\n\\n**Input**: \\\"Microsoft in Talks to Acquire Gaming Company for $2 Billion\\\" \\n**Output**: \\\"Business\\\"\\n\\n**Input**: \\\"Manchester United Secures Win in Premier League Football Match\\\" \\n**Output**: \\\"Sports\\\" \\n\\n# Notes\\n\\n- If the headline appears to fit into more than one category, choose the most dominant theme.\\n- Keywords or phrases such as \\\"stocks\\\", \\\"company acquisition\\\", \\\"match\\\", or technological brands can be good indicators for classification.\\n\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"{{item.input}}\"\n }\n }\n ]\n },\n \"model\": \"gpt-4o-mini\",\n \"sampling_params\": {\n \"seed\": 42,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n \"max_completions_tokens\": 2048\n }\n },\n \"error\": null,\n \"metadata\": {}\n}\n" ToolSearchToolParam: properties: type: type: string enum: - tool_search description: The type of the tool. Always `tool_search`. default: tool_search x-stainless-const: true execution: $ref: '#/components/schemas/ToolSearchExecutionType' description: Whether tool search is executed by the server or by the client. description: anyOf: - type: string description: Description shown to the model for a client-executed tool search tool. - type: 'null' parameters: anyOf: - $ref: '#/components/schemas/EmptyModelParam' description: Parameter schema for a client-executed tool search tool. - type: 'null' type: object required: - type title: Tool search tool description: Hosted or BYOT tool search configuration for deferred tools. GrammarSyntax1: type: string enum: - lark - regex ChatCompletionTool: type: object title: Function tool description: 'A function tool that can be used to generate a response. ' properties: type: type: string enum: - function description: The type of the tool. Currently, only `function` is supported. x-stainless-const: true function: $ref: '#/components/schemas/FunctionObject' required: - type - function Filters: anyOf: - $ref: '#/components/schemas/ComparisonFilter' - $ref: '#/components/schemas/CompoundFilter' GraderLabelModel: type: object title: LabelModelGrader description: 'A LabelModelGrader object which uses a model to assign labels to each item in the evaluation. ' properties: type: description: The object type, which is always `label_model`. type: string enum: - label_model x-stainless-const: true name: type: string description: The name of the grader. model: type: string description: The model to use for the evaluation. Must support structured outputs. input: type: array items: $ref: '#/components/schemas/EvalItem' labels: type: array items: type: string description: The labels to assign to each item in the evaluation. passing_labels: type: array items: type: string description: The labels that indicate a passing result. Must be a subset of labels. required: - type - model - input - passing_labels - labels - name x-oaiMeta: name: Label Model Grader group: graders example: "{\n \"name\": \"First label grader\",\n \"type\": \"label_model\",\n \"model\": \"gpt-4o-2024-08-06\",\n \"input\": [\n {\n \"type\": \"message\",\n \"role\": \"system\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Classify the sentiment of the following statement as one of positive, neutral, or negative\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Statement: {{item.response}}\"\n }\n }\n ],\n \"passing_labels\": [\n \"positive\"\n ],\n \"labels\": [\n \"positive\",\n \"neutral\",\n \"negative\"\n ]\n}\n" GraderPython: type: object title: PythonGrader description: 'A PythonGrader object that runs a python script on the input. ' properties: type: type: string enum: - python description: The object type, which is always `python`. x-stainless-const: true name: type: string description: The name of the grader. source: type: string description: The source code of the python script. image_tag: type: string description: The image tag to use for the python script. required: - type - name - source x-oaiMeta: name: Python Grader group: graders example: "{\n \"type\": \"python\",\n \"name\": \"Example python grader\",\n \"image_tag\": \"2025-05-08\",\n \"source\": \"\"\"\ndef grade(sample: dict, item: dict) -> float:\n \\\"\"\"\n Returns 1.0 if `output_text` equals `label`, otherwise 0.0.\n \\\"\"\"\n output = sample.get(\"output_text\")\n label = item.get(\"label\")\n return 1.0 if output == label else 0.0\n\"\"\",\n}\n" CreateEvalStoredCompletionsDataSourceConfig: type: object title: StoredCompletionsDataSourceConfig description: 'Deprecated in favor of LogsDataSourceConfig. ' properties: type: type: string enum: - stored_completions default: stored_completions description: The type of data source. Always `stored_completions`. x-stainless-const: true metadata: type: object description: Metadata filters for the stored completions data source. additionalProperties: true example: "{\n \"use_case\": \"customer_support_agent\"\n}\n" required: - type deprecated: true x-oaiMeta: name: The stored completions data source object for evals group: evals example: "{\n \"type\": \"stored_completions\",\n \"metadata\": {\n \"use_case\": \"customer_support_agent\"\n }\n}\n" ContainerNetworkPolicyDisabledParam: properties: type: type: string enum: - disabled description: Disable outbound network access. Always `disabled`. default: disabled x-stainless-const: true type: object required: - type ContainerAutoParam: properties: type: type: string enum: - container_auto description: Automatically creates a container for this request default: container_auto x-stainless-const: true file_ids: items: type: string example: file-123 type: array maxItems: 50 description: An optional list of uploaded files to make available to your code. memory_limit: anyOf: - $ref: '#/components/schemas/ContainerMemoryLimit' description: The memory limit for the container. - type: 'null' network_policy: oneOf: - $ref: '#/components/schemas/ContainerNetworkPolicyDisabledParam' - $ref: '#/components/schemas/ContainerNetworkPolicyAllowlistParam' description: Network access policy for the container. discriminator: propertyName: type skills: items: oneOf: - $ref: '#/components/schemas/SkillReferenceParam' - $ref: '#/components/schemas/InlineSkillParam' discriminator: propertyName: type type: array maxItems: 200 description: An optional list of skills referenced by id or inline data. type: object required: - type RankerVersionType: type: string enum: - auto - default-2024-11-15 MessagePhase: type: string description: 'Labels an `assistant` message as intermediate commentary (`commentary`) or the final answer (`final_answer`). For models like `gpt-5.3-codex` and beyond, when sending follow-up requests, preserve and resend phase on all assistant messages — dropping it can degrade performance. Not used for user messages. ' enum: - commentary - final_answer ResponseFormatJsonObject: type: object title: JSON object description: 'JSON object response format. An older method of generating JSON responses. Using `json_schema` is recommended for models that support it. Note that the model will not generate JSON without a system or user message instructing it to do so. ' properties: type: type: string description: The type of response format being defined. Always `json_object`. enum: - json_object x-stainless-const: true required: - type EvalCustomDataSourceConfig: type: object title: CustomDataSourceConfig description: 'A CustomDataSourceConfig which specifies the schema of your `item` and optionally `sample` namespaces. The response schema defines the shape of the data that will be: - Used to define your testing criteria and - What data is required when creating a run ' properties: type: type: string enum: - custom default: custom description: The type of data source. Always `custom`. x-stainless-const: true schema: type: object description: 'The json schema for the run data source items. Learn how to build JSON schemas [here](https://json-schema.org/). ' additionalProperties: true example: "{\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\",\n \"properties\": {\n \"label\": {\"type\": \"string\"},\n },\n \"required\": [\"label\"]\n }\n },\n \"required\": [\"item\"]\n}\n" required: - type - schema x-oaiMeta: name: The eval custom data source config object group: evals example: "{\n \"type\": \"custom\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\",\n \"properties\": {\n \"label\": {\"type\": \"string\"},\n },\n \"required\": [\"label\"]\n }\n },\n \"required\": [\"item\"]\n }\n}\n" EvalItem: type: object title: Eval message object description: 'A message input to the model with a role indicating instruction following hierarchy. Instructions given with the `developer` or `system` role take precedence over instructions given with the `user` role. Messages with the `assistant` role are presumed to have been generated by the model in previous interactions. ' properties: role: type: string description: 'The role of the message input. One of `user`, `assistant`, `system`, or `developer`. ' enum: - user - assistant - system - developer content: $ref: '#/components/schemas/EvalItemContent' type: type: string description: 'The type of the message input. Always `message`. ' enum: - message x-stainless-const: true required: - role - content FunctionParameters: type: object description: "The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format. \n\nOmitting `parameters` defines a function with an empty parameter list." additionalProperties: true EmptyModelParam: properties: {} type: object required: [] InputMessageContentList: type: array title: Input item content list description: "A list of one or many input items to the model, containing different content \ntypes.\n" items: $ref: '#/components/schemas/InputContent' ComputerUsePreviewTool: properties: type: type: string enum: - computer_use_preview description: The type of the computer use tool. Always `computer_use_preview`. default: computer_use_preview x-stainless-const: true environment: $ref: '#/components/schemas/ComputerEnvironment' description: The type of computer environment to control. display_width: type: integer description: The width of the computer display. display_height: type: integer description: The height of the computer display. type: object required: - type - environment - display_width - display_height title: Computer use preview description: A tool that controls a virtual computer. Learn more about the [computer tool](https://platform.openai.com/docs/guides/tools-computer-use). InlineSkillSourceParam: properties: type: type: string enum: - base64 description: The type of the inline skill source. Must be `base64`. default: base64 x-stainless-const: true media_type: type: string enum: - application/zip description: The media type of the inline skill payload. Must be `application/zip`. default: application/zip x-stainless-const: true data: type: string maxLength: 70254592 minLength: 1 description: Base64-encoded skill zip bundle. type: object required: - type - media_type - data description: Inline skill payload ApplyPatchToolParam: properties: type: type: string enum: - apply_patch description: The type of the tool. Always `apply_patch`. default: apply_patch x-stainless-const: true type: object required: - type title: Apply patch tool description: Allows the assistant to create, delete, or update files using unified diffs. CreateEvalRunRequest: type: object title: CreateEvalRunRequest properties: name: type: string description: The name of the run. metadata: $ref: '#/components/schemas/Metadata' data_source: type: object description: Details about the run's data source. oneOf: - $ref: '#/components/schemas/CreateEvalJsonlRunDataSource' - $ref: '#/components/schemas/CreateEvalCompletionsRunDataSource' - $ref: '#/components/schemas/CreateEvalResponsesRunDataSource' required: - data_source ComparisonFilter: type: object additionalProperties: false title: Comparison Filter description: 'A filter used to compare a specified attribute key to a given value using a defined comparison operation. ' properties: type: type: string default: eq enum: - eq - ne - gt - gte - lt - lte - in - nin description: 'Specifies the comparison operator: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`. - `eq`: equals - `ne`: not equal - `gt`: greater than - `gte`: greater than or equal - `lt`: less than - `lte`: less than or equal - `in`: in - `nin`: not in ' key: type: string description: The key to compare against the value. value: oneOf: - type: string - type: number - type: boolean - type: array items: oneOf: - type: string - type: number description: The value to compare against the attribute key; supports string, number, or boolean types. required: - type - key - value x-oaiMeta: name: ComparisonFilter LocalSkillParam: properties: name: type: string description: The name of the skill. description: type: string description: The description of the skill. path: type: string description: The path to the directory containing the skill. type: object required: - name - description - path ComputerEnvironment: type: string enum: - windows - mac - linux - ubuntu - browser EvalResponsesSource: type: object title: EvalResponsesSource description: 'A EvalResponsesSource object describing a run data source configuration. ' properties: type: type: string enum: - responses description: The type of run data source. Always `responses`. metadata: anyOf: - type: object description: Metadata filter for the responses. This is a query parameter used to select responses. - type: 'null' model: anyOf: - type: string description: The name of the model to find responses for. This is a query parameter used to select responses. - type: 'null' instructions_search: anyOf: - type: string description: Optional string to search the 'instructions' field. This is a query parameter used to select responses. - type: 'null' created_after: anyOf: - type: integer minimum: 0 description: Only include items created after this timestamp (inclusive). This is a query parameter used to select responses. - type: 'null' created_before: anyOf: - type: integer minimum: 0 description: Only include items created before this timestamp (inclusive). This is a query parameter used to select responses. - type: 'null' reasoning_effort: anyOf: - $ref: '#/components/schemas/ReasoningEffort' description: Optional reasoning effort parameter. This is a query parameter used to select responses. - type: 'null' temperature: anyOf: - type: number description: Sampling temperature. This is a query parameter used to select responses. - type: 'null' top_p: anyOf: - type: number description: Nucleus sampling parameter. This is a query parameter used to select responses. - type: 'null' users: anyOf: - type: array items: type: string description: List of user identifiers. This is a query parameter used to select responses. - type: 'null' tools: anyOf: - type: array items: type: string description: List of tool names. This is a query parameter used to select responses. - type: 'null' required: - type x-oaiMeta: name: The run data source object used to configure an individual run group: eval runs example: "{\n \"type\": \"responses\",\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"temperature\": 0.7,\n \"top_p\": 1.0,\n \"users\": [\"user1\", \"user2\"],\n \"tools\": [\"tool1\", \"tool2\"],\n \"instructions_search\": \"You are a coding assistant\"\n}\n" ContainerNetworkPolicyAllowlistParam: properties: type: type: string enum: - allowlist description: Allow outbound network access only to specified domains. Always `allowlist`. default: allowlist x-stainless-const: true allowed_domains: items: type: string type: array minItems: 1 description: A list of allowed domains when type is `allowlist`. domain_secrets: items: $ref: '#/components/schemas/ContainerNetworkPolicyDomainSecretParam' type: array minItems: 1 description: Optional domain-scoped secrets for allowlisted domains. type: object required: - type - allowed_domains MCPToolFilter: type: object title: MCP tool filter description: 'A filter object to specify which tools are allowed. ' properties: tool_names: type: array title: MCP allowed tools items: type: string description: List of allowed tool names. read_only: type: boolean description: 'Indicates whether or not a tool modifies data or is read-only. If an MCP server is [annotated with `readOnlyHint`](https://modelcontextprotocol.io/specification/2025-06-18/schema#toolannotations-readonlyhint), it will match this filter. ' required: [] additionalProperties: false EvalApiError: type: object title: EvalApiError description: 'An object representing an error response from the Eval API. ' properties: code: type: string description: The error code. message: type: string description: The error message. required: - code - message x-oaiMeta: name: The API error object group: evals example: "{\n \"code\": \"internal_error\",\n \"message\": \"The eval run failed due to an internal error.\"\n}\n" NamespaceToolParam: properties: type: type: string enum: - namespace description: The type of the tool. Always `namespace`. default: namespace x-stainless-const: true name: type: string minLength: 1 description: The namespace name used in tool calls (for example, `crm`). description: type: string minLength: 1 description: A description of the namespace shown to the model. tools: items: oneOf: - $ref: '#/components/schemas/FunctionToolParam' - $ref: '#/components/schemas/CustomToolParam' description: A function or custom tool that belongs to a namespace. discriminator: propertyName: type type: array minItems: 1 description: The function/custom tools available inside this namespace. type: object required: - type - name - description - tools title: Namespace description: Groups function/custom tools under a shared namespace. EvalItemContentItem: title: Eval content item description: 'A single content item: input text, output text, input image, or input audio. ' oneOf: - $ref: '#/components/schemas/EvalItemContentText' - $ref: '#/components/schemas/InputTextContent' - $ref: '#/components/schemas/EvalItemContentOutputText' - $ref: '#/components/schemas/EvalItemInputImage' - $ref: '#/components/schemas/InputAudio' ResponseFormatText: type: object title: Text description: 'Default response format. Used to generate text responses. ' properties: type: type: string description: The type of response format being defined. Always `text`. enum: - text x-stainless-const: true required: - type EasyInputMessage: type: object title: Input message description: 'A message input to the model with a role indicating instruction following hierarchy. Instructions given with the `developer` or `system` role take precedence over instructions given with the `user` role. Messages with the `assistant` role are presumed to have been generated by the model in previous interactions. ' properties: role: type: string description: 'The role of the message input. One of `user`, `assistant`, `system`, or `developer`. ' enum: - user - assistant - system - developer content: description: 'Text, image, or audio input to the model, used to generate a response. Can also contain previous assistant responses. ' oneOf: - type: string title: Text input description: 'A text input to the model. ' - $ref: '#/components/schemas/InputMessageContentList' phase: anyOf: - $ref: '#/components/schemas/MessagePhase' - type: 'null' type: type: string description: 'The type of the message input. Always `message`. ' enum: - message x-stainless-const: true required: - role - content Tool: description: 'A tool that can be used to generate a response. ' discriminator: propertyName: type oneOf: - $ref: '#/components/schemas/FunctionTool' - $ref: '#/components/schemas/FileSearchTool' - $ref: '#/components/schemas/ComputerTool' - $ref: '#/components/schemas/ComputerUsePreviewTool' - $ref: '#/components/schemas/WebSearchTool' - $ref: '#/components/schemas/MCPTool' - $ref: '#/components/schemas/CodeInterpreterTool' - $ref: '#/components/schemas/ImageGenTool' - $ref: '#/components/schemas/LocalShellToolParam' - $ref: '#/components/schemas/FunctionShellToolParam' - $ref: '#/components/schemas/CustomToolParam' - $ref: '#/components/schemas/NamespaceToolParam' - $ref: '#/components/schemas/ToolSearchToolParam' - $ref: '#/components/schemas/WebSearchPreviewTool' - $ref: '#/components/schemas/ApplyPatchToolParam' LocalShellToolParam: properties: type: type: string enum: - local_shell description: The type of the local shell tool. Always `local_shell`. default: local_shell x-stainless-const: true type: object required: - type title: Local shell tool description: A tool that allows the model to execute shell commands in a local environment. EvalItemContent: title: Eval content description: 'Inputs to the model - can contain template strings. Supports text, output text, input images, and input audio, either as a single item or an array of items. ' oneOf: - $ref: '#/components/schemas/EvalItemContentItem' - $ref: '#/components/schemas/EvalItemContentArray' CustomTextFormatParam: properties: type: type: string enum: - text description: Unconstrained text format. Always `text`. default: text x-stainless-const: true type: object required: - type title: Text format description: Unconstrained free-form text. MCPTool: type: object title: MCP tool description: 'Give the model access to additional tools via remote Model Context Protocol (MCP) servers. [Learn more about MCP](/docs/guides/tools-remote-mcp). ' properties: type: type: string enum: - mcp description: The type of the MCP tool. Always `mcp`. x-stainless-const: true server_label: type: string description: 'A label for this MCP server, used to identify it in tool calls. ' server_url: type: string format: uri description: 'The URL for the MCP server. One of `server_url` or `connector_id` must be provided. ' connector_id: type: string enum: - connector_dropbox - connector_gmail - connector_googlecalendar - connector_googledrive - connector_microsoftteams - connector_outlookcalendar - connector_outlookemail - connector_sharepoint description: 'Identifier for service connectors, like those available in ChatGPT. One of `server_url` or `connector_id` must be provided. Learn more about service connectors [here](/docs/guides/tools-remote-mcp#connectors). Currently supported `connector_id` values are: - Dropbox: `connector_dropbox` - Gmail: `connector_gmail` - Google Calendar: `connector_googlecalendar` - Google Drive: `connector_googledrive` - Microsoft Teams: `connector_microsoftteams` - Outlook Calendar: `connector_outlookcalendar` - Outlook Email: `connector_outlookemail` - SharePoint: `connector_sharepoint` ' authorization: type: string description: 'An OAuth access token that can be used with a remote MCP server, either with a custom MCP server URL or a service connector. Your application must handle the OAuth authorization flow and provide the token here. ' server_description: type: string description: 'Optional description of the MCP server, used to provide more context. ' headers: anyOf: - type: object additionalProperties: type: string description: 'Optional HTTP headers to send to the MCP server. Use for authentication or other purposes. ' - type: 'null' allowed_tools: anyOf: - description: 'List of allowed tool names or a filter object. ' oneOf: - type: array title: MCP allowed tools description: A string array of allowed tool names items: type: string - $ref: '#/components/schemas/MCPToolFilter' - type: 'null' require_approval: anyOf: - description: Specify which of the MCP server's tools require approval. oneOf: - type: object title: MCP tool approval filter description: 'Specify which of the MCP server''s tools require approval. Can be `always`, `never`, or a filter object associated with tools that require approval. ' properties: always: $ref: '#/components/schemas/MCPToolFilter' never: $ref: '#/components/schemas/MCPToolFilter' additionalProperties: false - type: string title: MCP tool approval setting description: 'Specify a single approval policy for all tools. One of `always` or `never`. When set to `always`, all tools will require approval. When set to `never`, all tools will not require approval. ' enum: - always - never default: always - type: 'null' defer_loading: type: boolean description: 'Whether this MCP tool is deferred and discovered via tool search. ' required: - type - server_label InputImageContent: properties: type: type: string enum: - input_image description: The type of the input item. Always `input_image`. default: input_image x-stainless-const: true image_url: anyOf: - type: string format: uri description: The URL of the image to be sent to the model. A fully qualified URL or base64 encoded image in a data URL. - type: 'null' file_id: anyOf: - type: string description: The ID of the file to be sent to the model. - type: 'null' detail: $ref: '#/components/schemas/ImageDetail' description: The detail level of the image to be sent to the model. One of `high`, `low`, `auto`, or `original`. Defaults to `auto`. type: object required: - type - detail title: Input image description: An image input to the model. Learn about [image inputs](/docs/guides/vision). LocalEnvironmentParam: properties: type: type: string enum: - local description: Use a local computer environment. default: local x-stainless-const: true skills: items: $ref: '#/components/schemas/LocalSkillParam' type: array maxItems: 200 description: An optional list of skills. type: object required: - type SearchContextSize: type: string enum: - low - medium - high ApproximateLocation: properties: type: type: string enum: - approximate description: The type of location approximation. Always `approximate`. default: approximate x-stainless-const: true country: anyOf: - type: string description: The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of the user, e.g. `US`. - type: 'null' region: anyOf: - type: string description: Free text input for the region of the user, e.g. `California`. - type: 'null' city: anyOf: - type: string description: Free text input for the city of the user, e.g. `San Francisco`. - type: 'null' timezone: anyOf: - type: string description: The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the user, e.g. `America/Los_Angeles`. - type: 'null' type: object required: - type FunctionObject: type: object properties: description: type: string description: A description of what the function does, used by the model to choose when and how to call the function. name: type: string description: The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. parameters: $ref: '#/components/schemas/FunctionParameters' strict: anyOf: - type: boolean default: false description: Whether to enable strict schema adherence when generating the function call. If set to true, the model will follow the exact schema defined in the `parameters` field. Only a subset of JSON Schema is supported when `strict` is `true`. Learn more about Structured Outputs in the [function calling guide](/docs/guides/function-calling). - type: 'null' required: - name GraderScoreModel: type: object title: ScoreModelGrader description: 'A ScoreModelGrader object that uses a model to assign a score to the input. ' properties: type: type: string enum: - score_model description: The object type, which is always `score_model`. x-stainless-const: true name: type: string description: The name of the grader. model: type: string description: The model to use for the evaluation. sampling_params: type: object description: The sampling parameters for the model. properties: seed: anyOf: - type: integer description: 'A seed value to initialize the randomness, during sampling. ' - type: 'null' top_p: anyOf: - type: number default: 1 example: 1 description: 'An alternative to temperature for nucleus sampling; 1.0 includes all tokens. ' - type: 'null' temperature: anyOf: - type: number description: 'A higher temperature increases randomness in the outputs. ' - type: 'null' max_completions_tokens: anyOf: - type: integer minimum: 1 description: 'The maximum number of tokens the grader model may generate in its response. ' - type: 'null' reasoning_effort: $ref: '#/components/schemas/ReasoningEffort' input: type: array items: $ref: '#/components/schemas/EvalItem' description: 'The input messages evaluated by the grader. Supports text, output text, input image, and input audio content blocks, and may include template strings. ' range: type: array items: type: number min_items: 2 max_items: 2 description: The range of the score. Defaults to `[0, 1]`. required: - type - name - input - model x-oaiMeta: name: Score Model Grader group: graders example: "{\n \"type\": \"score_model\",\n \"name\": \"Example score model grader\",\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": (\n \"Score how close the reference answer is to the model answer. Score 1.0 if they are the same and 0.0 if they are different.\"\n \" Return just a floating point score\\n\\n\"\n \" Reference answer: {{item.label}}\\n\\n\"\n \" Model answer: {{sample.output_text}}\"\n )\n },\n {\n \"type\": \"input_image\",\n \"image_url\": \"https://example.com/reference.png\",\n \"file_id\": null,\n \"detail\": \"auto\"\n }\n ],\n }\n ],\n \"model\": \"gpt-5-mini\",\n \"sampling_params\": {\n \"temperature\": 1,\n \"top_p\": 1,\n \"seed\": 42,\n \"max_completions_tokens\": 32768,\n \"reasoning_effort\": \"medium\"\n },\n}\n" InputTextContent: properties: type: type: string enum: - input_text description: The type of the input item. Always `input_text`. default: input_text x-stainless-const: true text: type: string description: The text input to the model. type: object required: - type - text title: Input text description: A text input to the model. EvalGraderTextSimilarity: type: object title: TextSimilarityGrader allOf: - $ref: '#/components/schemas/GraderTextSimilarity' - type: object properties: pass_threshold: type: number description: The threshold for the score. required: - pass_threshold x-oaiMeta: name: Text Similarity Grader group: graders example: "{\n \"type\": \"text_similarity\",\n \"name\": \"Example text similarity grader\",\n \"input\": \"{{sample.output_text}}\",\n \"reference\": \"{{item.label}}\",\n \"pass_threshold\": 0.8,\n \"evaluation_metric\": \"fuzzy_match\"\n}\n" EvalGraderLabelModel: type: object title: LabelModelGrader allOf: - $ref: '#/components/schemas/GraderLabelModel' EvalItemContentOutputText: type: object title: Output text description: 'A text output from the model. ' properties: type: type: string description: 'The type of the output text. Always `output_text`. ' enum: - output_text x-stainless-const: true text: type: string description: 'The text output from the model. ' required: - type - text RankingOptions: properties: ranker: $ref: '#/components/schemas/RankerVersionType' description: The ranker to use for the file search. score_threshold: type: number description: The score threshold for the file search, a number between 0 and 1. Numbers closer to 1 will attempt to return only the most relevant results, but may return fewer results. hybrid_search: $ref: '#/components/schemas/HybridSearchOptions' description: Weights that control how reciprocal rank fusion balances semantic embedding matches versus sparse keyword matches when hybrid search is enabled. type: object required: [] Error: type: object properties: code: anyOf: - type: string - type: 'null' message: type: string param: anyOf: - type: string - type: 'null' type: type: string required: - type - message - param - code FileSearchTool: properties: type: type: string enum: - file_search description: The type of the file search tool. Always `file_search`. default: file_search x-stainless-const: true vector_store_ids: items: type: string type: array description: The IDs of the vector stores to search. max_num_results: type: integer description: The maximum number of results to return. This number should be between 1 and 50 inclusive. ranking_options: $ref: '#/components/schemas/RankingOptions' description: Ranking options for search. filters: anyOf: - $ref: '#/components/schemas/Filters' description: A filter to apply. - type: 'null' type: object required: - type - vector_store_ids title: File search description: A tool that searches for relevant content from uploaded files. Learn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search). EvalStoredCompletionsDataSourceConfig: type: object title: StoredCompletionsDataSourceConfig description: 'Deprecated in favor of LogsDataSourceConfig. ' properties: type: type: string enum: - stored_completions default: stored_completions description: The type of data source. Always `stored_completions`. x-stainless-const: true metadata: $ref: '#/components/schemas/Metadata' schema: type: object description: 'The json schema for the run data source items. Learn how to build JSON schemas [here](https://json-schema.org/). ' additionalProperties: true required: - type - schema deprecated: true x-oaiMeta: name: The stored completions data source object for evals group: evals example: "{\n \"type\": \"stored_completions\",\n \"metadata\": {\n \"language\": \"english\"\n },\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\"\n },\n \"sample\": {\n \"type\": \"object\"\n }\n },\n \"required\": [\n \"item\",\n \"sample\"\n }\n}\n" ContainerMemoryLimit: type: string enum: - 1g - 4g - 16g - 64g GraderStringCheck: type: object title: StringCheckGrader description: 'A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. ' properties: type: type: string enum: - string_check description: The object type, which is always `string_check`. x-stainless-const: true name: type: string description: The name of the grader. input: type: string description: The input text. This may include template strings. reference: type: string description: The reference text. This may include template strings. operation: type: string enum: - eq - ne - like - ilike description: The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. required: - type - name - input - reference - operation x-oaiMeta: name: String Check Grader group: graders example: "{\n \"type\": \"string_check\",\n \"name\": \"Example string check grader\",\n \"input\": \"{{sample.output_text}}\",\n \"reference\": \"{{item.label}}\",\n \"operation\": \"eq\"\n}\n" CreateEvalCustomDataSourceConfig: type: object title: CustomDataSourceConfig description: 'A CustomDataSourceConfig object that defines the schema for the data source used for the evaluation runs. This schema is used to define the shape of the data that will be: - Used to define your testing criteria and - What data is required when creating a run ' properties: type: type: string enum: - custom default: custom description: The type of data source. Always `custom`. x-stainless-const: true item_schema: type: object description: The json schema for each row in the data source. additionalProperties: true example: "{\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"age\": {\"type\": \"integer\"}\n },\n \"required\": [\"name\", \"age\"]\n}\n" include_sample_schema: type: boolean default: false description: Whether the eval should expect you to populate the sample namespace (ie, by generating responses off of your data source) required: - item_schema - type x-oaiMeta: name: The eval file data source config object group: evals example: "{\n \"type\": \"custom\",\n \"item_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"age\": {\"type\": \"integer\"}\n },\n \"required\": [\"name\", \"age\"]\n },\n \"include_sample_schema\": true\n}\n" FunctionShellToolParam: properties: type: type: string enum: - shell description: The type of the shell tool. Always `shell`. default: shell x-stainless-const: true environment: anyOf: - oneOf: - $ref: '#/components/schemas/ContainerAutoParam' - $ref: '#/components/schemas/LocalEnvironmentParam' - $ref: '#/components/schemas/ContainerReferenceParam' discriminator: propertyName: type - type: 'null' type: object required: - type title: Shell tool description: A tool that allows the model to execute shell commands. SkillReferenceParam: properties: type: type: string enum: - skill_reference description: References a skill created with the /v1/skills endpoint. default: skill_reference x-stainless-const: true skill_id: type: string maxLength: 64 minLength: 1 description: The ID of the referenced skill. version: type: string description: Optional skill version. Use a positive integer or 'latest'. Omit for default. type: object required: - type - skill_id EvalItemContentText: type: string title: Text input description: 'A text input to the model. ' InputContent: oneOf: - $ref: '#/components/schemas/InputTextContent' - $ref: '#/components/schemas/InputImageContent' - $ref: '#/components/schemas/InputFileContent' discriminator: propertyName: type CreateEvalCompletionsRunDataSource: type: object title: CompletionsRunDataSource description: 'A CompletionsRunDataSource object describing a model sampling configuration. ' properties: type: type: string enum: - completions default: completions description: The type of run data source. Always `completions`. input_messages: description: Used when sampling from a model. Dictates the structure of the messages passed into the model. Can either be a reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template with variable references to the `item` namespace. oneOf: - type: object title: TemplateInputMessages properties: type: type: string enum: - template description: The type of input messages. Always `template`. template: type: array description: A list of chat messages forming the prompt or context. May include variable references to the `item` namespace, ie {{item.name}}. items: oneOf: - $ref: '#/components/schemas/EasyInputMessage' - $ref: '#/components/schemas/EvalItem' required: - type - template - type: object title: ItemReferenceInputMessages properties: type: type: string enum: - item_reference description: The type of input messages. Always `item_reference`. item_reference: type: string description: A reference to a variable in the `item` namespace. Ie, "item.input_trajectory" required: - type - item_reference sampling_params: type: object properties: reasoning_effort: $ref: '#/components/schemas/ReasoningEffort' temperature: type: number description: A higher temperature increases randomness in the outputs. default: 1 max_completion_tokens: type: integer description: The maximum number of tokens in the generated output. top_p: type: number description: An alternative to temperature for nucleus sampling; 1.0 includes all tokens. default: 1 seed: type: integer description: A seed value to initialize the randomness, during sampling. default: 42 response_format: description: 'An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. Learn more in the [Structured Outputs guide](/docs/guides/structured-outputs). Setting to `{ "type": "json_object" }` enables the older JSON mode, which ensures the message the model generates is valid JSON. Using `json_schema` is preferred for models that support it. ' oneOf: - $ref: '#/components/schemas/ResponseFormatText' - $ref: '#/components/schemas/ResponseFormatJsonSchema' - $ref: '#/components/schemas/ResponseFormatJsonObject' tools: type: array description: 'A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. ' items: $ref: '#/components/schemas/ChatCompletionTool' model: type: string description: The name of the model to use for generating completions (e.g. "o3-mini"). source: description: Determines what populates the `item` namespace in this run's data source. oneOf: - $ref: '#/components/schemas/EvalJsonlFileContentSource' - $ref: '#/components/schemas/EvalJsonlFileIdSource' - $ref: '#/components/schemas/EvalStoredCompletionsSource' required: - type - source x-oaiMeta: name: The completions data source object used to configure an individual run group: eval runs example: "{\n \"name\": \"gpt-4o-mini-2024-07-18\",\n \"data_source\": {\n \"type\": \"completions\",\n \"input_messages\": {\n \"type\": \"item_reference\",\n \"item_reference\": \"item.input\"\n },\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"source\": {\n \"type\": \"stored_completions\",\n \"model\": \"gpt-4o-mini-2024-07-18\"\n }\n }\n}\n" WebSearchPreviewTool: properties: type: type: string enum: - web_search_preview - web_search_preview_2025_03_11 description: The type of the web search tool. One of `web_search_preview` or `web_search_preview_2025_03_11`. default: web_search_preview x-stainless-const: true user_location: anyOf: - $ref: '#/components/schemas/ApproximateLocation' description: The user's location. - type: 'null' search_context_size: $ref: '#/components/schemas/SearchContextSize' description: High level guidance for the amount of context window space to use for the search. One of `low`, `medium`, or `high`. `medium` is the default. search_content_types: items: $ref: '#/components/schemas/SearchContentType' type: array type: object required: - type title: Web search preview description: This tool searches the web for relevant results to use in a response. Learn more about the [web search tool](https://platform.openai.com/docs/guides/tools-web-search). EvalRunOutputItem: type: object title: EvalRunOutputItem description: 'A schema representing an evaluation run output item. ' properties: object: type: string enum: - eval.run.output_item default: eval.run.output_item description: The type of the object. Always "eval.run.output_item". x-stainless-const: true id: type: string description: Unique identifier for the evaluation run output item. run_id: type: string description: The identifier of the evaluation run associated with this output item. eval_id: type: string description: The identifier of the evaluation group. created_at: type: integer format: unixtime description: Unix timestamp (in seconds) when the evaluation run was created. status: type: string description: The status of the evaluation run. datasource_item_id: type: integer description: The identifier for the data source item. datasource_item: type: object description: Details of the input data source item. additionalProperties: true results: type: array description: A list of grader results for this output item. items: $ref: '#/components/schemas/EvalRunOutputItemResult' sample: type: object description: A sample containing the input and output of the evaluation run. properties: input: type: array description: An array of input messages. items: type: object description: An input message. properties: role: type: string description: The role of the message sender (e.g., system, user, developer). content: type: string description: The content of the message. required: - role - content output: type: array description: An array of output messages. items: type: object properties: role: type: string description: The role of the message (e.g. "system", "assistant", "user"). content: type: string description: The content of the message. finish_reason: type: string description: The reason why the sample generation was finished. model: type: string description: The model used for generating the sample. usage: type: object description: Token usage details for the sample. properties: total_tokens: type: integer description: The total number of tokens used. completion_tokens: type: integer description: The number of completion tokens generated. prompt_tokens: type: integer description: The number of prompt tokens used. cached_tokens: type: integer description: The number of tokens retrieved from cache. required: - total_tokens - completion_tokens - prompt_tokens - cached_tokens error: $ref: '#/components/schemas/EvalApiError' temperature: type: number description: The sampling temperature used. max_completion_tokens: type: integer description: The maximum number of tokens allowed for completion. top_p: type: number description: The top_p value used for sampling. seed: type: integer description: The seed used for generating the sample. required: - input - output - finish_reason - model - usage - error - temperature - max_completion_tokens - top_p - seed required: - object - id - run_id - eval_id - created_at - status - datasource_item_id - datasource_item - results - sample x-oaiMeta: name: The eval run output item object group: evals example: "{\n \"object\": \"eval.run.output_item\",\n \"id\": \"outputitem_67abd55eb6548190bb580745d5644a33\",\n \"run_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"eval_id\": \"eval_67abd54d9b0081909a86353f6fb9317a\",\n \"created_at\": 1739314509,\n \"status\": \"pass\",\n \"datasource_item_id\": 137,\n \"datasource_item\": {\n \"teacher\": \"To grade essays, I only check for style, content, and grammar.\",\n \"student\": \"I am a student who is trying to write the best essay.\"\n },\n \"results\": [\n {\n \"name\": \"String Check Grader\",\n \"type\": \"string-check-grader\",\n \"score\": 1.0,\n \"passed\": true,\n }\n ],\n \"sample\": {\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an evaluator bot...\"\n },\n {\n \"role\": \"user\",\n \"content\": \"You are assessing...\"\n }\n ],\n \"output\": [\n {\n \"role\": \"assistant\",\n \"content\": \"The rubric is not clear nor concise.\"\n }\n ],\n \"finish_reason\": \"stop\",\n \"model\": \"gpt-4o-2024-08-06\",\n \"usage\": {\n \"total_tokens\": 521,\n \"completion_tokens\": 2,\n \"prompt_tokens\": 519,\n \"cached_tokens\": 0\n },\n \"error\": null,\n \"temperature\": 1.0,\n \"max_completion_tokens\": 2048,\n \"top_p\": 1.0,\n \"seed\": 42\n }\n}\n" CreateEvalRequest: type: object title: CreateEvalRequest properties: name: type: string description: The name of the evaluation. metadata: $ref: '#/components/schemas/Metadata' data_source_config: type: object description: The configuration for the data source used for the evaluation runs. Dictates the schema of the data used in the evaluation. oneOf: - $ref: '#/components/schemas/CreateEvalCustomDataSourceConfig' - $ref: '#/components/schemas/CreateEvalLogsDataSourceConfig' - $ref: '#/components/schemas/CreateEvalStoredCompletionsDataSourceConfig' testing_criteria: type: array description: A list of graders for all eval runs in this group. Graders can reference variables in the data source using double curly braces notation, like `{{item.variable_name}}`. To reference the model's output, use the `sample` namespace (ie, `{{sample.output_text}}`). items: oneOf: - $ref: '#/components/schemas/CreateEvalLabelModelGrader' - $ref: '#/components/schemas/EvalGraderStringCheck' - $ref: '#/components/schemas/EvalGraderTextSimilarity' - $ref: '#/components/schemas/EvalGraderPython' - $ref: '#/components/schemas/EvalGraderScoreModel' required: - data_source_config - testing_criteria CompoundFilter: $recursiveAnchor: true type: object additionalProperties: false title: Compound Filter description: Combine multiple filters using `and` or `or`. properties: type: type: string description: 'Type of operation: `and` or `or`.' enum: - and - or filters: type: array description: Array of filters to combine. Items can be `ComparisonFilter` or `CompoundFilter`. items: oneOf: - $ref: '#/components/schemas/ComparisonFilter' - $recursiveRef: '#' discriminator: propertyName: type required: - type - filters x-oaiMeta: name: CompoundFilter CodeInterpreterTool: type: object title: Code interpreter description: 'A tool that runs Python code to help generate a response to a prompt. ' properties: type: type: string enum: - code_interpreter description: 'The type of the code interpreter tool. Always `code_interpreter`. ' x-stainless-const: true container: description: 'The code interpreter container. Can be a container ID or an object that specifies uploaded file IDs to make available to your code, along with an optional `memory_limit` setting. ' oneOf: - type: string description: The container ID. - $ref: '#/components/schemas/AutoCodeInterpreterToolParam' required: - type - container CreateEvalJsonlRunDataSource: type: object title: JsonlRunDataSource description: "A JsonlRunDataSource object with that specifies a JSONL file that matches the eval \n" properties: type: type: string enum: - jsonl default: jsonl description: The type of data source. Always `jsonl`. x-stainless-const: true source: description: Determines what populates the `item` namespace in the data source. oneOf: - $ref: '#/components/schemas/EvalJsonlFileContentSource' - $ref: '#/components/schemas/EvalJsonlFileIdSource' required: - type - source x-oaiMeta: name: The file data source object for the eval run configuration group: evals example: "{\n \"type\": \"jsonl\",\n \"source\": {\n \"type\": \"file_id\",\n \"id\": \"file-9GYS6xbkWgWhmE7VoLUWFg\"\n }\n}\n" ComputerTool: properties: type: type: string enum: - computer description: The type of the computer tool. Always `computer`. default: computer x-stainless-const: true type: object required: - type title: Computer description: A tool that controls a virtual computer. Learn more about the [computer tool](https://platform.openai.com/docs/guides/tools-computer-use). EvalJsonlFileContentSource: type: object title: EvalJsonlFileContentSource properties: type: type: string enum: - file_content default: file_content description: The type of jsonl source. Always `file_content`. x-stainless-const: true content: type: array items: type: object properties: item: type: object additionalProperties: true sample: type: object additionalProperties: true required: - item description: The content of the jsonl file. required: - type - content EvalItemContentArray: type: array title: An array of Input text, Output text, Input image, and Input audio description: 'A list of inputs, each of which may be either an input text, output text, input image, or input audio object. ' items: $ref: '#/components/schemas/EvalItemContentItem' AutoCodeInterpreterToolParam: properties: type: type: string enum: - auto description: Always `auto`. default: auto x-stainless-const: true file_ids: items: type: string example: file-123 type: array maxItems: 50 description: An optional list of uploaded files to make available to your code. memory_limit: anyOf: - $ref: '#/components/schemas/ContainerMemoryLimit' description: The memory limit for the code interpreter container. - type: 'null' network_policy: oneOf: - $ref: '#/components/schemas/ContainerNetworkPolicyDisabledParam' - $ref: '#/components/schemas/ContainerNetworkPolicyAllowlistParam' description: Network access policy for the container. discriminator: propertyName: type type: object required: - type title: CodeInterpreterToolAuto description: Configuration for a code interpreter container. Optionally specify the IDs of the files to run the code on. EvalGraderPython: type: object title: PythonGrader allOf: - $ref: '#/components/schemas/GraderPython' - type: object properties: pass_threshold: type: number description: The threshold for the score. x-oaiMeta: name: Eval Python Grader group: graders example: "{\n \"type\": \"python\",\n \"name\": \"Example python grader\",\n \"image_tag\": \"2025-05-08\",\n \"source\": \"\"\"\ndef grade(sample: dict, item: dict) -> float:\n \\\"\"\"\n Returns 1.0 if `output_text` equals `label`, otherwise 0.0.\n \\\"\"\"\n output = sample.get(\"output_text\")\n label = item.get(\"label\")\n return 1.0 if output == label else 0.0\n\"\"\",\n \"pass_threshold\": 0.8\n}\n" CustomToolParam: properties: type: type: string enum: - custom description: The type of the custom tool. Always `custom`. default: custom x-stainless-const: true name: type: string description: The name of the custom tool, used to identify it in tool calls. description: type: string description: Optional description of the custom tool, used to provide more context. format: oneOf: - $ref: '#/components/schemas/CustomTextFormatParam' - $ref: '#/components/schemas/CustomGrammarFormatParam' description: The input format for the custom tool. Default is unconstrained text. discriminator: propertyName: type defer_loading: type: boolean description: Whether this tool should be deferred and discovered via tool search. type: object required: - type - name title: Custom tool description: A custom tool that processes input using a specified format. Learn more about [custom tools](/docs/guides/function-calling#custom-tools) InputAudio: type: object title: Input audio description: 'An audio input to the model. ' properties: type: type: string description: 'The type of the input item. Always `input_audio`. ' enum: - input_audio x-stainless-const: true input_audio: type: object properties: data: type: string description: 'Base64-encoded audio data. ' format: type: string description: 'The format of the audio data. Currently supported formats are `mp3` and `wav`. ' enum: - mp3 - wav required: - data - format required: - type - input_audio FunctionTool: properties: type: type: string enum: - function description: The type of the function tool. Always `function`. default: function x-stainless-const: true name: type: string description: The name of the function to call. description: anyOf: - type: string description: A description of the function. Used by the model to determine whether or not to call the function. - type: 'null' parameters: anyOf: - additionalProperties: {} type: object description: A JSON schema object describing the parameters of the function. x-oaiTypeLabel: map - type: 'null' strict: anyOf: - type: boolean description: Whether to enforce strict parameter validation. Default `true`. - type: 'null' defer_loading: type: boolean description: Whether this function is deferred and loaded via tool search. type: object required: - type - name - strict - parameters title: Function description: Defines a function in your own code the model can choose to call. Learn more about [function calling](https://platform.openai.com/docs/guides/function-calling). ContainerNetworkPolicyDomainSecretParam: properties: domain: type: string minLength: 1 description: The domain associated with the secret. name: type: string minLength: 1 description: The name of the secret to inject for the domain. value: type: string maxLength: 10485760 minLength: 1 description: The secret value to inject for the domain. type: object required: - domain - name - value InputFileContent: properties: type: type: string enum: - input_file description: The type of the input item. Always `input_file`. default: input_file x-stainless-const: true file_id: anyOf: - type: string description: The ID of the file to be sent to the model. - type: 'null' filename: type: string description: The name of the file to be sent to the model. file_data: type: string description: 'The content of the file to be sent to the model. ' file_url: type: string format: uri description: The URL of the file to be sent to the model. detail: $ref: '#/components/schemas/FileInputDetail' description: The detail level of the file to be sent to the model. Use `low` for the default rendering behavior, or `high` to render the file at higher quality. Defaults to `low`. type: object required: - type title: Input file description: A file input to the model. HybridSearchOptions: properties: embedding_weight: type: number description: The weight of the embedding in the reciprocal ranking fusion. text_weight: type: number description: The weight of the text in the reciprocal ranking fusion. type: object required: - embedding_weight - text_weight CreateEvalResponsesRunDataSource: type: object title: ResponsesRunDataSource description: 'A ResponsesRunDataSource object describing a model sampling configuration. ' properties: type: type: string enum: - responses default: responses description: The type of run data source. Always `responses`. input_messages: description: Used when sampling from a model. Dictates the structure of the messages passed into the model. Can either be a reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template with variable references to the `item` namespace. oneOf: - type: object title: InputMessagesTemplate properties: type: type: string enum: - template description: The type of input messages. Always `template`. template: type: array description: A list of chat messages forming the prompt or context. May include variable references to the `item` namespace, ie {{item.name}}. items: oneOf: - type: object title: ChatMessage properties: role: type: string description: The role of the message (e.g. "system", "assistant", "user"). content: type: string description: The content of the message. required: - role - content - $ref: '#/components/schemas/EvalItem' required: - type - template - type: object title: InputMessagesItemReference properties: type: type: string enum: - item_reference description: The type of input messages. Always `item_reference`. item_reference: type: string description: A reference to a variable in the `item` namespace. Ie, "item.name" required: - type - item_reference sampling_params: type: object properties: reasoning_effort: $ref: '#/components/schemas/ReasoningEffort' temperature: type: number description: A higher temperature increases randomness in the outputs. default: 1 max_completion_tokens: type: integer description: The maximum number of tokens in the generated output. top_p: type: number description: An alternative to temperature for nucleus sampling; 1.0 includes all tokens. default: 1 seed: type: integer description: A seed value to initialize the randomness, during sampling. default: 42 tools: type: array description: "An array of tools the model may call while generating a response. You\ncan specify which tool to use by setting the `tool_choice` parameter.\n\nThe two categories of tools you can provide the model are:\n\n- **Built-in tools**: Tools that are provided by OpenAI that extend the\n model's capabilities, like [web search](/docs/guides/tools-web-search)\n or [file search](/docs/guides/tools-file-search). Learn more about\n [built-in tools](/docs/guides/tools).\n- **Function calls (custom tools)**: Functions that are defined by you,\n enabling the model to call your own code. Learn more about\n [function calling](/docs/guides/function-calling).\n" items: $ref: '#/components/schemas/Tool' text: type: object description: 'Configuration options for a text response from the model. Can be plain text or structured JSON data. Learn more: - [Text inputs and outputs](/docs/guides/text) - [Structured Outputs](/docs/guides/structured-outputs) ' properties: format: $ref: '#/components/schemas/TextResponseFormatConfiguration' model: type: string description: The name of the model to use for generating completions (e.g. "o3-mini"). source: description: Determines what populates the `item` namespace in this run's data source. oneOf: - $ref: '#/components/schemas/EvalJsonlFileContentSource' - $ref: '#/components/schemas/EvalJsonlFileIdSource' - $ref: '#/components/schemas/EvalResponsesSource' required: - type - source x-oaiMeta: name: The completions data source object used to configure an individual run group: eval runs example: "{\n \"name\": \"gpt-4o-mini-2024-07-18\",\n \"data_source\": {\n \"type\": \"responses\",\n \"input_messages\": {\n \"type\": \"item_reference\",\n \"item_reference\": \"item.input\"\n },\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"source\": {\n \"type\": \"responses\",\n \"model\": \"gpt-4o-mini-2024-07-18\"\n }\n }\n}\n" CreateEvalLogsDataSourceConfig: type: object title: LogsDataSourceConfig description: 'A data source config which specifies the metadata property of your logs query. This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. ' properties: type: type: string enum: - logs default: logs description: The type of data source. Always `logs`. x-stainless-const: true metadata: type: object description: Metadata filters for the logs data source. additionalProperties: true example: "{\n \"use_case\": \"customer_support_agent\"\n}\n" required: - type x-oaiMeta: name: The logs data source object for evals group: evals example: "{\n \"type\": \"logs\",\n \"metadata\": {\n \"use_case\": \"customer_support_agent\"\n }\n}\n" EvalRunList: type: object title: EvalRunList description: 'An object representing a list of runs for an evaluation. ' properties: object: type: string enum: - list default: list description: 'The type of this object. It is always set to "list". ' x-stainless-const: true data: type: array description: 'An array of eval run objects. ' items: $ref: '#/components/schemas/EvalRun' first_id: type: string description: The identifier of the first eval run in the data array. last_id: type: string description: The identifier of the last eval run in the data array. has_more: type: boolean description: Indicates whether there are more evals available. required: - object - data - first_id - last_id - has_more x-oaiMeta: name: The eval run list object group: evals example: "{\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"eval.run\",\n \"id\": \"evalrun_67b7fbdad46c819092f6fe7a14189620\",\n \"eval_id\": \"eval_67b7fa9a81a88190ab4aa417e397ea21\",\n \"report_url\": \"https://platform.openai.com/evaluations/eval_67b7fa9a81a88190ab4aa417e397ea21?run_id=evalrun_67b7fbdad46c819092f6fe7a14189620\",\n \"status\": \"completed\",\n \"model\": \"o3-mini\",\n \"name\": \"Academic Assistant\",\n \"created_at\": 1740110812,\n \"result_counts\": {\n \"total\": 171,\n \"errored\": 0,\n \"failed\": 80,\n \"passed\": 91\n },\n \"per_model_usage\": null,\n \"per_testing_criteria_results\": [\n {\n \"testing_criteria\": \"String check grader\",\n \"passed\": 91,\n \"failed\": 80\n }\n ],\n \"run_data_source\": {\n \"type\": \"completions\",\n \"template_messages\": [\n {\n \"type\": \"message\",\n \"role\": \"system\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"You are a helpful assistant.\"\n }\n },\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": {\n \"type\": \"input_text\",\n \"text\": \"Hello, can you help me with my homework?\"\n }\n }\n ],\n \"datasource_reference\": null,\n \"model\": \"o3-mini\",\n \"max_completion_tokens\": null,\n \"seed\": null,\n \"temperature\": null,\n \"top_p\": null\n },\n \"error\": null,\n \"metadata\": {\"test\": \"synthetics\"}\n }\n ],\n \"first_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"last_id\": \"evalrun_67abd54d60ec8190832b46859da808f7\",\n \"has_more\": false\n}\n" CreateEvalLabelModelGrader: type: object title: LabelModelGrader description: 'A LabelModelGrader object which uses a model to assign labels to each item in the evaluation. ' properties: type: description: The object type, which is always `label_model`. type: string enum: - label_model x-stainless-const: true name: type: string description: The name of the grader. model: type: string description: The model to use for the evaluation. Must support structured outputs. input: type: array description: A list of chat messages forming the prompt or context. May include variable references to the `item` namespace, ie {{item.name}}. items: $ref: '#/components/schemas/CreateEvalItem' labels: type: array items: type: string description: The labels to classify to each item in the evaluation. passing_labels: type: array items: type: string description: The labels that indicate a passing result. Must be a subset of labels. required: - type - model - input - passing_labels - labels - name x-oaiMeta: name: The eval label model grader object group: evals example: "{\n \"type\": \"label_model\",\n \"model\": \"gpt-4o-2024-08-06\",\n \"input\": [\n {\n \"role\": \"system\",\n \"content\": \"Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Statement: {{item.response}}\"\n }\n ],\n \"passing_labels\": [\"positive\"],\n \"labels\": [\"positive\", \"neutral\", \"negative\"],\n \"name\": \"Sentiment label grader\"\n}\n" EvalGraderStringCheck: type: object title: StringCheckGrader allOf: - $ref: '#/components/schemas/GraderStringCheck' ReasoningEffort: anyOf: - type: string enum: - none - minimal - low - medium - high - xhigh default: medium description: 'Constrains effort on reasoning for [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response. - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool calls are supported for all reasoning values in gpt-5.1. - All models before `gpt-5.1` default to `medium` reasoning effort, and do not support `none`. - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort. - `xhigh` is supported for all models after `gpt-5.1-codex-max`. ' - type: 'null' EvalLogsDataSourceConfig: type: object title: LogsDataSourceConfig description: 'A LogsDataSourceConfig which specifies the metadata property of your logs query. This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The schema returned by this data source config is used to defined what variables are available in your evals. `item` and `sample` are both defined when using this data source config. ' properties: type: type: string enum: - logs default: logs description: The type of data source. Always `logs`. x-stainless-const: true metadata: $ref: '#/components/schemas/Metadata' schema: type: object description: 'The json schema for the run data source items. Learn how to build JSON schemas [here](https://json-schema.org/). ' additionalProperties: true required: - type - schema x-oaiMeta: name: The logs data source object for evals group: evals example: "{\n \"type\": \"logs\",\n \"metadata\": {\n \"language\": \"english\"\n },\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"item\": {\n \"type\": \"object\"\n },\n \"sample\": {\n \"type\": \"object\"\n }\n },\n \"required\": [\n \"item\",\n \"sample\"\n }\n}\n" TextResponseFormatJsonSchema: type: object title: JSON schema description: 'JSON Schema response format. Used to generate structured JSON responses. Learn more about [Structured Outputs](/docs/guides/structured-outputs). ' properties: type: type: string description: The type of response format being defined. Always `json_schema`. enum: - json_schema x-stainless-const: true description: type: string description: 'A description of what the response format is for, used by the model to determine how to respond in the format. ' name: type: string description: 'The name of the response format. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. ' schema: $ref: '#/components/schemas/ResponseFormatJsonSchemaSchema' strict: anyOf: - type: boolean default: false description: 'Whether to enable strict schema adherence when generating the output. If set to true, the model will always follow the exact schema defined in the `schema` field. Only a subset of JSON Schema is supported when `strict` is `true`. To learn more, read the [Structured Outputs guide](/docs/guides/structured-outputs). ' - type: 'null' required: - type - schema - name ToolSearchExecutionType: type: string enum: - server - client FunctionToolParam: properties: name: type: string maxLength: 128 minLength: 1 pattern: ^[a-zA-Z0-9_-]+$ description: anyOf: - type: string - type: 'null' parameters: anyOf: - $ref: '#/components/schemas/EmptyModelParam' - type: 'null' strict: anyOf: - type: boolean - type: 'null' type: type: string enum: - function default: function x-stainless-const: true defer_loading: type: boolean description: Whether this function should be deferred and discovered via tool search. type: object required: - name - type EvalStoredCompletionsSource: type: object title: StoredCompletionsRunDataSource description: 'A StoredCompletionsRunDataSource configuration describing a set of filters ' properties: type: type: string enum: - stored_completions default: stored_completions description: The type of source. Always `stored_completions`. x-stainless-const: true metadata: $ref: '#/components/schemas/Metadata' model: anyOf: - type: string description: An optional model to filter by (e.g., 'gpt-4o'). - type: 'null' created_after: anyOf: - type: integer description: An optional Unix timestamp to filter items created after this time. - type: 'null' created_before: anyOf: - type: integer description: An optional Unix timestamp to filter items created before this time. - type: 'null' limit: anyOf: - type: integer description: An optional maximum number of items to return. - type: 'null' required: - type x-oaiMeta: name: The stored completions data source object used to configure an individual run group: eval runs example: "{\n \"type\": \"stored_completions\",\n \"model\": \"gpt-4o\",\n \"created_after\": 1668124800,\n \"created_before\": 1668124900,\n \"limit\": 100,\n \"metadata\": {}\n}\n" GraderTextSimilarity: type: object title: TextSimilarityGrader description: 'A TextSimilarityGrader object which grades text based on similarity metrics. ' properties: type: type: string enum: - text_similarity default: text_similarity description: The type of grader. x-stainless-const: true name: type: string description: The name of the grader. input: type: string description: The text being graded. reference: type: string description: The text being graded against. evaluation_metric: type: string enum: - cosine - fuzzy_match - bleu - gleu - meteor - rouge_1 - rouge_2 - rouge_3 - rouge_4 - rouge_5 - rouge_l description: "The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, \n`gleu`, `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, \nor `rouge_l`.\n" required: - type - name - input - reference - evaluation_metric x-oaiMeta: name: Text Similarity Grader group: graders example: "{\n \"type\": \"text_similarity\",\n \"name\": \"Example text similarity grader\",\n \"input\": \"{{sample.output_text}}\",\n \"reference\": \"{{item.label}}\",\n \"evaluation_metric\": \"fuzzy_match\"\n}\n" Metadata: anyOf: - type: object description: 'Set of 16 key-value pairs that can be attached to an object. This can be useful for storing additional information about the object in a structured format, and querying for objects via API or the dashboard. Keys are strings with a maximum length of 64 characters. Values are strings with a maximum length of 512 characters. ' additionalProperties: type: string x-oaiTypeLabel: map - type: 'null' InputFidelity: type: string enum: - high - low description: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported for `gpt-image-1` and `gpt-image-1.5` and later models, unsupported for `gpt-image-1-mini`. Supports `high` and `low`. Defaults to `low`. TextResponseFormatConfiguration: description: "An object specifying the format that the model must output.\n\nConfiguring `{ \"type\": \"json_schema\" }` enables Structured Outputs, \nwhich ensures the model will match your supplied JSON schema. Learn more in the \n[Structured Outputs guide](/docs/guides/structured-outputs).\n\nThe default format is `{ \"type\": \"text\" }` with no additional options.\n\n**Not recommended for gpt-4o and newer models:**\n\nSetting to `{ \"type\": \"json_object\" }` enables the older JSON mode, which\nensures the message the model generates is valid JSON. Using `json_schema`\nis preferred for models that support it.\n" oneOf: - $ref: '#/components/schemas/ResponseFormatText' - $ref: '#/components/schemas/TextResponseFormatJsonSchema' - $ref: '#/components/schemas/ResponseFormatJsonObject' ImageDetail: type: string enum: - low - high - auto - original WebSearchApproximateLocation: anyOf: - type: object title: Web search approximate location description: 'The approximate location of the user. ' properties: type: type: string enum: - approximate description: The type of location approximation. Always `approximate`. default: approximate x-stainless-const: true country: anyOf: - type: string description: The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of the user, e.g. `US`. - type: 'null' region: anyOf: - type: string description: Free text input for the region of the user, e.g. `California`. - type: 'null' city: anyOf: - type: string description: Free text input for the city of the user, e.g. `San Francisco`. - type: 'null' timezone: anyOf: - type: string description: The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the user, e.g. `America/Los_Angeles`. - type: 'null' - type: 'null' EvalJsonlFileIdSource: type: object title: EvalJsonlFileIdSource properties: type: type: string enum: - file_id default: file_id description: The type of jsonl source. Always `file_id`. x-stainless-const: true id: type: string description: The identifier of the file. required: - type - id ResponseFormatJsonSchemaSchema: type: object title: JSON schema description: 'The schema for the response format, described as a JSON Schema object. Learn how to build JSON schemas [here](https://json-schema.org/). ' additionalProperties: true ContainerReferenceParam: properties: type: type: string enum: - container_reference description: References a container created with the /v1/containers endpoint default: container_reference x-stainless-const: true container_id: type: string description: The ID of the referenced container. example: cntr_123 type: object required: - type - container_id CreateEvalItem: title: CreateEvalItem description: A chat message that makes up the prompt or context. May include variable references to the `item` namespace, ie {{item.name}}. type: object oneOf: - type: object title: SimpleInputMessage properties: role: type: string description: The role of the message (e.g. "system", "assistant", "user"). content: type: string description: The content of the message. required: - role - content - $ref: '#/components/schemas/EvalItem' x-oaiMeta: name: The chat message object used to configure an individual run CustomGrammarFormatParam: properties: type: type: string enum: - grammar description: Grammar format. Always `grammar`. default: grammar x-stainless-const: true syntax: $ref: '#/components/schemas/GrammarSyntax1' description: The syntax of the grammar definition. One of `lark` or `regex`. definition: type: string description: The grammar definition. type: object required: - type - syntax - definition title: Grammar format description: A grammar defined by the user. ImageGenTool: type: object title: Image generation tool description: 'A tool that generates images using the GPT image models. ' properties: type: type: string enum: - image_generation description: 'The type of the image generation tool. Always `image_generation`. ' x-stainless-const: true model: anyOf: - type: string - type: string enum: - gpt-image-1 - gpt-image-1-mini - gpt-image-1.5 description: 'The image generation model to use. Default: `gpt-image-1`. ' default: gpt-image-1 quality: type: string enum: - low - medium - high - auto description: 'The quality of the generated image. One of `low`, `medium`, `high`, or `auto`. Default: `auto`. ' default: auto size: anyOf: - type: string - type: string enum: - 1024x1024 - 1024x1536 - 1536x1024 - auto description: The size of the generated images. For `gpt-image-2` and `gpt-image-2-2026-04-21`, arbitrary resolutions are supported as `WIDTHxHEIGHT` strings, for example `1536x864`. Width and height must both be divisible by 16 and the requested aspect ratio must be between 1:3 and 3:1. Resolutions above `2560x1440` are experimental, and the maximum supported resolution is `3840x2160`. The requested size must also satisfy the model's current pixel and edge limits. The standard sizes `1024x1024`, `1536x1024`, and `1024x1536` are supported by the GPT image models; `auto` is supported for models that allow automatic sizing. For `dall-e-2`, use one of `256x256`, `512x512`, or `1024x1024`. For `dall-e-3`, use one of `1024x1024`, `1792x1024`, or `1024x1792`. default: auto output_format: type: string enum: - png - webp - jpeg description: 'The output format of the generated image. One of `png`, `webp`, or `jpeg`. Default: `png`. ' default: png output_compression: type: integer minimum: 0 maximum: 100 description: 'Compression level for the output image. Default: 100. ' default: 100 moderation: type: string enum: - auto - low description: 'Moderation level for the generated image. Default: `auto`. ' default: auto background: type: string enum: - transparent - opaque - auto description: 'Background type for the generated image. One of `transparent`, `opaque`, or `auto`. Default: `auto`. ' default: auto input_fidelity: anyOf: - $ref: '#/components/schemas/InputFidelity' - type: 'null' input_image_mask: type: object description: 'Optional mask for inpainting. Contains `image_url` (string, optional) and `file_id` (string, optional). ' properties: image_url: type: string description: 'Base64-encoded mask image. ' file_id: type: string description: 'File ID for the mask image. ' required: [] additionalProperties: false partial_images: type: integer minimum: 0 maximum: 3 description: 'Number of partial images to generate in streaming mode, from 0 (default value) to 3. ' default: 0 action: description: 'Whether to generate a new image or edit an existing image. Default: `auto`. ' $ref: '#/components/schemas/ImageGenActionEnum' required: - type securitySchemes: ApiKeyAuth: type: http scheme: bearer x-oaiMeta: groups: - id: audio title: Audio description: 'Learn how to turn audio into text or text into audio. Related guide: [Speech to text](/docs/guides/speech-to-text) ' sections: - type: endpoint key: createSpeech path: createSpeech - type: endpoint key: createTranscription path: createTranscription - type: endpoint key: createTranslation path: createTranslation - id: chat title: Chat description: 'Given a list of messages comprising a conversation, the model will return a response. Related guide: [Chat Completions](/docs/guides/text-generation) ' sections: - type: endpoint key: createChatCompletion path: create - type: object key: CreateChatCompletionResponse path: object - type: object key: CreateChatCompletionStreamResponse path: streaming - id: embeddings title: Embeddings description: 'Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. Related guide: [Embeddings](/docs/guides/embeddings) ' sections: - type: endpoint key: createEmbedding path: create - type: object key: Embedding path: object - id: fine-tuning title: Fine-tuning description: 'Manage fine-tuning jobs to tailor a model to your specific training data. Related guide: [Fine-tune models](/docs/guides/fine-tuning) ' sections: - type: endpoint key: createFineTuningJob path: create - type: endpoint key: listPaginatedFineTuningJobs path: list - type: endpoint key: listFineTuningEvents path: list-events - type: endpoint key: retrieveFineTuningJob path: retrieve - type: endpoint key: cancelFineTuningJob path: cancel - type: object key: FineTuningJob path: object - type: object key: FineTuningJobEvent path: event-object - id: files title: Files description: 'Files are used to upload documents that can be used with features like [Assistants](/docs/api-reference/assistants) and [Fine-tuning](/docs/api-reference/fine-tuning). ' sections: - type: endpoint key: createFile path: create - type: endpoint key: listFiles path: list - type: endpoint key: retrieveFile path: retrieve - type: endpoint key: deleteFile path: delete - type: endpoint key: downloadFile path: retrieve-contents - type: object key: OpenAIFile path: object - id: images title: Images description: 'Given a prompt and/or an input image, the model will generate a new image. Related guide: [Image generation](/docs/guides/images) ' sections: - type: endpoint key: createImage path: create - type: endpoint key: createImageEdit path: createEdit - type: endpoint key: createImageVariation path: createVariation - type: object key: Image path: object - id: models title: Models description: 'List and describe the various models available in the API. You can refer to the [Models](/docs/models) documentation to understand what models are available and the differences between them. ' sections: - type: endpoint key: listModels path: list - type: endpoint key: retrieveModel path: retrieve - type: endpoint key: deleteModel path: delete - type: object key: Model path: object - id: moderations title: Moderations description: 'Given a input text, outputs if the model classifies it as violating OpenAI''s content policy. Related guide: [Moderations](/docs/guides/moderation) ' sections: - type: endpoint key: createModeration path: create - type: object key: CreateModerationResponse path: object - id: assistants title: Assistants beta: true description: 'Build assistants that can call models and use tools to perform tasks. [Get started with the Assistants API](/docs/assistants) ' sections: - type: endpoint key: createAssistant path: createAssistant - type: endpoint key: createAssistantFile path: createAssistantFile - type: endpoint key: listAssistants path: listAssistants - type: endpoint key: listAssistantFiles path: listAssistantFiles - type: endpoint key: getAssistant path: getAssistant - type: endpoint key: getAssistantFile path: getAssistantFile - type: endpoint key: modifyAssistant path: modifyAssistant - type: endpoint key: deleteAssistant path: deleteAssistant - type: endpoint key: deleteAssistantFile path: deleteAssistantFile - type: object key: AssistantObject path: object - type: object key: AssistantFileObject path: file-object - id: threads title: Threads beta: true description: 'Create threads that assistants can interact with. Related guide: [Assistants](/docs/assistants/overview) ' sections: - type: endpoint key: createThread path: createThread - type: endpoint key: getThread path: getThread - type: endpoint key: modifyThread path: modifyThread - type: endpoint key: deleteThread path: deleteThread - type: object key: ThreadObject path: object - id: messages title: Messages beta: true description: 'Create messages within threads Related guide: [Assistants](/docs/assistants/overview) ' sections: - type: endpoint key: createMessage path: createMessage - type: endpoint key: listMessages path: listMessages - type: endpoint key: listMessageFiles path: listMessageFiles - type: endpoint key: getMessage path: getMessage - type: endpoint key: getMessageFile path: getMessageFile - type: endpoint key: modifyMessage path: modifyMessage - type: object key: MessageObject path: object - type: object key: MessageFileObject path: file-object - id: runs title: Runs beta: true description: 'Represents an execution run on a thread. Related guide: [Assistants](/docs/assistants/overview) ' sections: - type: endpoint key: createRun path: createRun - type: endpoint key: createThreadAndRun path: createThreadAndRun - type: endpoint key: listRuns path: listRuns - type: endpoint key: listRunSteps path: listRunSteps - type: endpoint key: getRun path: getRun - type: endpoint key: getRunStep path: getRunStep - type: endpoint key: modifyRun path: modifyRun - type: endpoint key: submitToolOuputsToRun path: submitToolOutputs - type: endpoint key: cancelRun path: cancelRun - type: object key: RunObject path: object - type: object key: RunStepObject path: step-object - id: completions title: Completions legacy: true description: 'Given a prompt, the model will return one or more predicted completions along with the probabilities of alternative tokens at each position. Most developer should use our [Chat Completions API](/docs/guides/text-generation/text-generation-models) to leverage our best and newest models. Most models that support the legacy Completions endpoint [will be shut off on January 4th, 2024](/docs/deprecations/2023-07-06-gpt-and-embeddings). ' sections: - type: endpoint key: createCompletion path: create - type: object key: CreateCompletionResponse path: object