openapi: 3.0.3 info: title: remediation.proto Audio evaluation API version: version not set servers: - url: https://api.together.xyz/v1 security: - bearerAuth: [] tags: - name: evaluation paths: /evaluation: post: tags: - evaluation summary: Create an evaluation job operationId: createEvaluationJob x-codeSamples: - lang: Python label: Together AI SDK (v2) source: "# Docs for v1 can be found by changing the above selector ^\nfrom together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evals.create(\n type=\"classify\",\n parameters=ParametersEvaluationClassifyParameters(\n judge=ParametersEvaluationClassifyParametersJudge(\n model=\"openai/gpt-oss-120b\",\n model_source=\"serverless\",\n system_template=\"You are an expert evaluator...\",\n ),\n input_data_file_path=\"file-abc123\",\n labels=[\"good\", \"bad\"],\n pass_labels=[\"good\"],\n model_to_evaluate=\"Qwen/Qwen3.5-9B\"\n )\n)\n\nprint(response.workflow_id)\n" - lang: Python label: Together AI SDK (v1) source: "from together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evaluation.create(\n type=\"classify\",\n judge_model_name=\"openai/gpt-oss-120b\",\n judge_system_template=\"You are an expert evaluator...\",\n input_data_file_path=\"file-abc123\",\n labels=[\"good\", \"bad\"],\n pass_labels=[\"good\"],\n model_to_evaluate=\"Qwen/Qwen3.5-9B\"\n)\n\nprint(response.workflow_id)\n" - lang: TypeScript label: Together AI SDK (TypeScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.create({\n type: 'classify',\n parameters: {\n judge: {\n model: 'openai/gpt-oss-120b',\n model_source: 'serverless',\n system_template: 'You are an expert evaluator...',\n },\n input_data_file_path: 'file-abc123',\n labels: ['good', 'bad'],\n pass_labels: ['good'],\n model_to_evaluate: 'Qwen/Qwen3.5-9B',\n },\n});\n\nconsole.log(response.workflow_id);\n" - lang: JavaScript label: Together AI SDK (JavaScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.create({\n type: 'classify',\n parameters: {\n judge: {\n model: 'openai/gpt-oss-120b',\n model_source: 'serverless',\n system_template: 'You are an expert evaluator...',\n },\n input_data_file_path: 'file-abc123',\n labels: ['good', 'bad'],\n pass_labels: ['good'],\n model_to_evaluate: 'Qwen/Qwen3.5-9B',\n },\n});\n\nconsole.log(response.workflow_id);\n" requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/EvaluationTypedRequest' responses: '200': description: Evaluation job created successfully content: application/json: schema: $ref: '#/components/schemas/EvaluationResponse' '400': description: Invalid request format content: application/json: schema: $ref: '#/components/schemas/ErrorData' '500': description: Failed to create evaluation job content: application/json: schema: $ref: '#/components/schemas/ErrorData' get: tags: - evaluation summary: Get all evaluation jobs operationId: getAllEvaluationJobs x-codeSamples: - lang: Python label: Together AI SDK (v2) source: "# Docs for v1 can be found by changing the above selector ^\nfrom together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evals.list()\n\nfor job in response:\n print(job.workflow_id)\n" - lang: Python label: Together AI SDK (v1) source: "from together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\njobs = client.evaluation.list()\n\nfor job in jobs:\n print(job.workflow_id)\n" - lang: TypeScript label: Together AI SDK (TypeScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.list();\n\nfor (const job of response) {\n console.log(job.workflow_id);\n}\n" - lang: JavaScript label: Together AI SDK (JavaScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.list();\n\nfor (const job of response) {\n console.log(job.workflow_id);\n}\n" parameters: - name: status in: query required: false schema: type: string description: Filter evaluation jobs by status - name: limit in: query required: false schema: type: integer default: 10 description: Limit the number of results responses: '200': description: evaluation jobs retrieved successfully content: application/json: schema: type: array items: $ref: '#/components/schemas/EvaluationJob' '400': description: Invalid request format content: application/json: schema: $ref: '#/components/schemas/ErrorData' '500': description: Error retrieving jobs from manager content: application/json: schema: $ref: '#/components/schemas/ErrorData' /evaluation/model-list: get: tags: - evaluation summary: Get model list operationId: getModelList parameters: - name: model_source in: query required: false schema: type: string description: Filter models by source default: all responses: '200': description: Model list retrieved successfully content: application/json: schema: type: object properties: model_list: type: array items: type: string description: The name of the model '400': description: Invalid request format content: application/json: schema: $ref: '#/components/schemas/ErrorData' '500': description: Error retrieving model list content: application/json: schema: $ref: '#/components/schemas/ErrorData' /evaluation/{id}: get: tags: - evaluation summary: Get evaluation job details operationId: getEvaluationJobDetails x-codeSamples: - lang: Python label: Together AI SDK (v2) source: "# Docs for v1 can be found by changing the above selector ^\nfrom together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evals.retrieve('eval_id')\n\nprint(response)\n" - lang: Python label: Together AI SDK (v1) source: "from together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evaluation.retrieve('eval_id')\n\nprint(response)\n" - lang: TypeScript label: Together AI SDK (TypeScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.retrieve('eval_id');\n\nconsole.log(response);\n" - lang: JavaScript label: Together AI SDK (JavaScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.retrieve('eval_id');\n\nconsole.log(response);\n" parameters: - name: id in: path required: true schema: description: The ID of the evaluation job to retrieve type: string responses: '200': description: Evaluation job details retrieved successfully content: application/json: schema: $ref: '#/components/schemas/EvaluationJob' '404': description: Evaluation job not found content: application/json: schema: $ref: '#/components/schemas/ErrorData' '500': description: Failed to get evaluation job content: application/json: schema: $ref: '#/components/schemas/ErrorData' /evaluation/{id}/status: get: tags: - evaluation summary: Get evaluation job status and results operationId: getEvaluationJobStatusAndResults x-codeSamples: - lang: Python label: Together AI SDK (v2) source: "# Docs for v1 can be found by changing the above selector ^\nfrom together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evals.status('eval_id')\n\nprint(response.status)\nprint(response.results)\n" - lang: Python label: Together AI SDK (v1) source: "from together import Together\nimport os\n\nclient = Together(\n api_key=os.environ.get(\"TOGETHER_API_KEY\"),\n)\n\nresponse = client.evaluation.status('eval_id')\n\nprint(response.status)\nprint(response.results)\n" - lang: TypeScript label: Together AI SDK (TypeScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.status('eval_id');\n\nconsole.log(response.status);\nconsole.log(response.results);\n" - lang: JavaScript label: Together AI SDK (JavaScript) source: "import Together from \"together-ai\";\n\nconst client = new Together({\n apiKey: process.env.TOGETHER_API_KEY,\n});\n\nconst response = await client.evals.status('eval_id');\n\nconsole.log(response.status);\nconsole.log(response.results);\n" parameters: - name: id in: path required: true schema: description: The ID of the evaluation job to get the status of type: string responses: '200': description: Evaluation job status and results retrieved successfully content: application/json: schema: type: object properties: status: type: string description: The status of the evaluation job enum: - completed - error - user_error - running - queued - pending results: description: The results of the evaluation job oneOf: - $ref: '#/components/schemas/EvaluationClassifyResults' - $ref: '#/components/schemas/EvaluationScoreResults' - $ref: '#/components/schemas/EvaluationCompareResults' '404': description: Evaluation job not found content: application/json: schema: $ref: '#/components/schemas/ErrorData' '500': description: Failed to get evaluation job content: application/json: schema: $ref: '#/components/schemas/ErrorData' components: schemas: EvaluationJobStatusUpdate: type: object properties: status: type: string description: The status at this update example: pending message: type: string description: Additional message for this update example: Job is pending evaluation timestamp: type: string format: date-time description: When this update occurred example: '2025-07-23T17:10:04.837888Z' EvaluationClassifyResults: type: object properties: generation_fail_count: type: number format: integer nullable: true description: Number of failed generations. example: 0 judge_fail_count: type: number format: integer nullable: true description: Number of failed judge generations example: 0 invalid_label_count: type: number format: float nullable: true description: Number of invalid labels example: 0 result_file_id: type: string description: Data File ID example: file-1234-aefd pass_percentage: type: number format: integer nullable: true description: Pecentage of pass labels. example: 10 label_counts: type: string description: JSON string representing label counts example: '{"yes": 10, "no": 0}' EvaluationCompareResults: type: object properties: num_samples: type: integer description: Total number of samples compared A_wins: type: integer description: Number of times model A won B_wins: type: integer description: Number of times model B won Ties: type: integer description: Number of ties generation_fail_count: type: number format: integer nullable: true description: Number of failed generations. example: 0 judge_fail_count: type: number format: integer nullable: true description: Number of failed judge generations example: 0 result_file_id: type: string description: Data File ID EvaluationScoreResults: type: object properties: aggregated_scores: type: object properties: mean_score: type: number format: float std_score: type: number format: float pass_percentage: type: number format: float generation_fail_count: type: number format: integer nullable: true description: Number of failed generations. example: 0 judge_fail_count: type: number format: integer nullable: true description: Number of failed judge generations example: 0 invalid_score_count: type: number format: integer description: number of invalid scores generated from model failed_samples: type: number format: integer description: number of failed samples generated from model result_file_id: type: string description: Data File ID example: file-1234-aefd EvaluationCompareParameters: type: object required: - judge - input_data_file_path properties: judge: $ref: '#/components/schemas/EvaluationJudgeModelConfig' model_a: $ref: '#/components/schemas/EvaluationModelOrString' model_b: $ref: '#/components/schemas/EvaluationModelOrString' input_data_file_path: type: string description: Data file name ErrorData: type: object required: - error properties: error: type: object properties: message: type: string nullable: false type: type: string nullable: false param: type: string nullable: true default: null code: type: string nullable: true default: null required: - type - message EvaluationModelRequest: type: object required: - model - max_tokens - temperature - system_template - input_template - model_source properties: model: type: string description: Name of the model to evaluate example: Qwen/Qwen3.5-9B max_tokens: type: integer minimum: 1 description: Maximum number of tokens to generate example: 512 temperature: type: number format: float minimum: 0 maximum: 2 description: Sampling temperature example: 0.7 system_template: type: string description: System prompt template example: Imagine you are helpful assistant input_template: type: string description: Input prompt template example: Please classify {{prompt}} based on the labels below model_source: type: string description: Source of the model. enum: - serverless - dedicated - external external_api_token: type: string description: Bearer/API token for external models. external_base_url: type: string description: Base URL for external models. Must be OpenAI-compatible base URL num_workers: type: integer minimum: 1 description: Number of concurrent workers for inference requests. Overrides the default concurrency for this model. Useful for tuning throughput when using proxy endpoints (e.g. OpenRouter) or rate-limited external APIs. example: 5 EvaluationScoreParameters: type: object required: - judge - min_score - max_score - pass_threshold - input_data_file_path properties: judge: $ref: '#/components/schemas/EvaluationJudgeModelConfig' min_score: type: number format: float example: 0 description: Minimum possible score max_score: type: number format: float example: 10 description: Maximum possible score pass_threshold: type: number format: float example: 7 description: Score threshold for passing model_to_evaluate: $ref: '#/components/schemas/EvaluationModelOrString' input_data_file_path: type: string example: file-01234567890123456789 description: Data file ID EvaluationJudgeModelConfig: type: object required: - model - system_template - model_source properties: model: type: string description: Name of the judge model example: Qwen/Qwen3.5-9B system_template: type: string description: System prompt template for the judge example: Imagine you are a helpful assistant model_source: type: string description: Source of the judge model. enum: - serverless - dedicated - external external_api_token: type: string description: Bearer/API token for external judge models. external_base_url: type: string description: Base URL for external judge models. Must be OpenAI-compatible base URL. num_workers: type: integer minimum: 1 description: Number of concurrent workers for inference requests. Overrides the default concurrency for this model. Useful for tuning throughput when using proxy endpoints (e.g. OpenRouter) or rate-limited external APIs. example: 5 max_tokens: type: integer minimum: 1 description: Maximum number of tokens the judge model can generate. Defaults to 32768. Increase for reasoning models (e.g. Gemini, o-series) that consume output token budget for chain-of-thought. example: 8192 temperature: type: number format: float minimum: 0 maximum: 2 description: Sampling temperature for the judge model. Defaults to 0.05. example: 0 EvaluationTypedRequest: type: object required: - type - parameters properties: type: type: string enum: - classify - score - compare description: The type of evaluation to perform example: classify parameters: oneOf: - $ref: '#/components/schemas/EvaluationClassifyParameters' - $ref: '#/components/schemas/EvaluationScoreParameters' - $ref: '#/components/schemas/EvaluationCompareParameters' description: Type-specific parameters for the evaluation EvaluationResponse: type: object properties: workflow_id: type: string description: The ID of the created evaluation job example: eval-1234-1244513 status: type: string enum: - pending description: Initial status of the job EvaluationClassifyParameters: type: object required: - judge - labels - pass_labels - input_data_file_path properties: judge: $ref: '#/components/schemas/EvaluationJudgeModelConfig' labels: type: array items: type: string minItems: 2 description: List of possible classification labels example: - 'yes' - 'no' pass_labels: type: array items: type: string minItems: 1 description: List of labels that are considered passing example: - 'yes' model_to_evaluate: $ref: '#/components/schemas/EvaluationModelOrString' input_data_file_path: type: string description: Data file ID example: file-1234-aefd EvaluationJob: type: object properties: workflow_id: type: string description: The evaluation job ID example: eval-1234aedf type: type: string enum: - classify - score - compare description: The type of evaluation example: classify owner_id: type: string description: ID of the job owner (admin only) status: type: string enum: - pending - queued - running - completed - error - user_error description: Current status of the job example: completed status_updates: type: array items: $ref: '#/components/schemas/EvaluationJobStatusUpdate' description: History of status updates (admin only) parameters: type: object description: The parameters used for this evaluation additionalProperties: true created_at: type: string format: date-time description: When the job was created example: '2025-07-23T17:10:04.837888Z' updated_at: type: string format: date-time description: When the job was last updated example: '2025-07-23T17:10:04.837888Z' results: oneOf: - $ref: '#/components/schemas/EvaluationClassifyResults' - $ref: '#/components/schemas/EvaluationScoreResults' - $ref: '#/components/schemas/EvaluationCompareResults' - type: object properties: error: type: string nullable: true description: Results of the evaluation (when completed) EvaluationModelOrString: oneOf: - type: string description: Field name in the input data - $ref: '#/components/schemas/EvaluationModelRequest' securitySchemes: bearerAuth: type: http scheme: bearer x-bearer-format: bearer x-default: default