openapi: 3.2.0 info: title: Airia Web Agent Evaluation API version: 1.0.0 servers: - url: https://api.airia.ai tags: - name: AgentEvaluation paths: /v1/AgentEvaluation/{id}: get: tags: - AgentEvaluation operationId: AgentEvaluation_Get parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/GetAgentEvaluationResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get x-summary-source: derived delete: tags: - AgentEvaluation operationId: AgentEvaluation_DeleteAgentEvaluation parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 204: description: '' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 422: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation delete agent evaluation x-summary-source: derived /v1/AgentEvaluation/airia-evaluators: get: tags: - AgentEvaluation operationId: AgentEvaluation_ListAiriaEvaluators parameters: - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: type: array items: $ref: '#/components/schemas/AiriaEvaluatorView' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation list airia evaluators x-summary-source: derived /v1/AgentEvaluation/Clone/{id}: post: tags: - AgentEvaluation operationId: AgentEvaluation_Clone parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: acknowledgeFoundrySkipped in: query schema: type: boolean x-position: 2 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/GetAgentEvaluationResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 409: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation clone x-summary-source: derived /v1/AgentEvaluation/Results: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetEvaluationsResult parameters: - name: PageNumber in: query schema: type: integer format: int32 x-position: 1 - name: PageSize in: query schema: type: integer format: int32 x-position: 2 - name: SortBy in: query schema: type: - string - 'null' x-position: 3 - name: SortDirection in: query schema: type: string x-position: 4 - name: filter in: query schema: type: - string - 'null' x-position: 5 - name: agentId in: query schema: type: - string - 'null' format: guid x-position: 6 - name: spmAgentId in: query schema: type: - string - 'null' format: guid x-position: 7 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/PagedResultOfAgentEvaluationDto' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get evaluations result x-summary-source: derived /v1/AgentEvaluation/target-summary: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetEvaluationTargetSummary parameters: - name: agentId in: query schema: type: - string - 'null' format: guid x-position: 1 - name: spmAgentId in: query schema: type: - string - 'null' format: guid x-position: 2 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/AgentEvaluationTargetSummaryDto' 400: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get evaluation target summary x-summary-source: derived /v1/AgentEvaluation/AggregatedResults/{evaluationJobId}: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetAggregatedResults parameters: - name: evaluationJobId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/AgentEvaluationAggregatedResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get aggregated results x-summary-source: derived /v1/AgentEvaluation/DetailedResults/{evaluationJobId}/{agentVersionId}: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetDetailedResults parameters: - name: evaluationJobId in: path required: true schema: type: string format: guid x-position: 1 - name: agentVersionId in: path required: true schema: type: string format: guid x-position: 2 - name: resultsType in: query schema: default: FullDetails oneOf: - $ref: '#/components/schemas/EvaluationResultsType' x-position: 3 - name: PageNumber in: query schema: type: integer format: int32 x-position: 4 - name: PageSize in: query schema: type: integer format: int32 x-position: 5 - name: SortBy in: query schema: type: - string - 'null' x-position: 6 - name: SortDirection in: query schema: type: string x-position: 7 - name: filter in: query schema: type: - string - 'null' x-position: 8 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/AgentEvaluationDetailedResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get detailed results x-summary-source: derived /v1/AgentEvaluation/DetailedResults/{evaluationJobId}/{agentVersionId}/export/csv: get: tags: - AgentEvaluation operationId: AgentEvaluation_ExportDetailedResultsAsCsv parameters: - name: evaluationJobId in: path required: true schema: type: string format: guid x-position: 1 - name: agentVersionId in: path required: true schema: type: string format: guid x-position: 2 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/octet-stream: schema: type: string format: binary 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation export detailed results as csv x-summary-source: derived /v1/AgentEvaluation/ComparisonResults/{evaluationJobId}: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetComparisonResults parameters: - name: evaluationJobId in: path required: true schema: type: string format: guid x-position: 1 - name: PageNumber in: query schema: type: integer format: int32 x-position: 2 - name: PageSize in: query schema: type: integer format: int32 x-position: 3 - name: SortBy in: query schema: type: - string - 'null' x-position: 4 - name: SortDirection in: query schema: type: string x-position: 5 - name: filter in: query schema: type: - string - 'null' x-position: 6 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/AgentEvaluationComparisonResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get comparison results x-summary-source: derived /v1/AgentEvaluation: post: tags: - AgentEvaluation operationId: AgentEvaluation_Create parameters: - name: x-correlation-id in: header schema: type: string requestBody: content: multipart/form-data: schema: type: object properties: EvaluationName: type: string Description: type: - string - 'null' AgentVersionIds: type: - array - 'null' items: type: string format: guid ExternalAgents: type: - array - 'null' items: $ref: '#/components/schemas/AgentCardModel' EvaluationType: $ref: '#/components/schemas/EvaluationType' ModelIdentifierType: $ref: '#/components/schemas/ModelIdentifierType' ModelId: type: - string - 'null' format: guid EvaluatorSource: oneOf: - $ref: '#/components/schemas/EvaluatorSource' SpmConnectionId: type: - string - 'null' format: guid Evaluators: type: - array - 'null' items: $ref: '#/components/schemas/EvaluatorSelectionInput' DatasetFile: type: - string - 'null' format: binary ManualTestCases: type: - array - 'null' items: $ref: '#/components/schemas/EvaluationTestCaseInput' HttpEndpointConfig.EndpointUrl: type: string HttpEndpointConfig.HttpMethod: type: string HttpEndpointConfig.Headers: type: - object - 'null' additionalProperties: type: string HttpEndpointConfig.HeadersJson: type: - string - 'null' HttpEndpointConfig.BodyTemplate: type: - string - 'null' HttpEndpointConfig.ResponseJsonPath: type: - string - 'null' HttpEndpointConfig.TimeoutMs: type: integer format: int32 HttpEndpointConfig.QueryStringsJson: type: - string - 'null' HttpEndpointConfig.IgnoreSslErrors: type: boolean HttpEndpointConfig.BodyType: type: - string - 'null' HttpEndpointConfig.CredentialId: type: - string - 'null' format: guid SpmAgentId: type: - string - 'null' format: guid SpmAliasId: type: - string - 'null' VariantsJson: type: - string - 'null' BaselineEvaluationId: type: - string - 'null' format: guid ConsistencyRunCount: type: - integer - 'null' format: int32 responses: 201: description: '' content: application/json: schema: $ref: '#/components/schemas/CreateAgentEvaluationResponse' 400: description: '' content: application/json: schema: type: string 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation create x-summary-source: derived /v1/AgentEvaluation/Dataset/{evaluationId}: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetDataset parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: PageNumber in: query schema: type: integer format: int32 x-position: 2 - name: PageSize in: query schema: type: integer format: int32 x-position: 3 - name: SortBy in: query schema: type: - string - 'null' x-position: 4 - name: SortDirection in: query schema: type: string x-position: 5 - name: filter in: query schema: type: - string - 'null' x-position: 6 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/PagedResultOfAgentEvaluationDatasetDto' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get dataset x-summary-source: derived /v1/AgentEvaluation/{evaluationId}/Schedule: post: tags: - AgentEvaluation operationId: AgentEvaluation_CreateSchedule parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string requestBody: x-name: request content: application/json: schema: $ref: '#/components/schemas/CreateEvaluationScheduleRequest' required: true x-position: 2 responses: 201: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationScheduleResponse' 400: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation create schedule x-summary-source: derived get: tags: - AgentEvaluation operationId: AgentEvaluation_GetSchedule parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationScheduleResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get schedule x-summary-source: derived put: tags: - AgentEvaluation operationId: AgentEvaluation_UpdateSchedule parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string requestBody: x-name: request content: application/json: schema: $ref: '#/components/schemas/UpdateEvaluationScheduleRequest' required: true x-position: 2 responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationScheduleResponse' 400: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation update schedule x-summary-source: derived delete: tags: - AgentEvaluation operationId: AgentEvaluation_DeleteSchedule parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 204: description: '' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation delete schedule x-summary-source: derived /v1/AgentEvaluation/{evaluationId}/Thresholds: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetThresholds parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationThresholdResponse' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get thresholds x-summary-source: derived put: tags: - AgentEvaluation operationId: AgentEvaluation_UpsertThresholds parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string requestBody: x-name: request content: application/json: schema: $ref: '#/components/schemas/UpsertEvaluationThresholdRequest' required: true x-position: 2 responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationThresholdResponse' 400: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation upsert thresholds x-summary-source: derived delete: tags: - AgentEvaluation operationId: AgentEvaluation_DeleteThresholds parameters: - name: evaluationId in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 204: description: '' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation delete thresholds x-summary-source: derived /v1/AgentEvaluation/{id}/run: post: tags: - AgentEvaluation operationId: AgentEvaluation_Run parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 202: description: '' content: application/json: schema: $ref: '#/components/schemas/RunAgentEvaluationResponse' 400: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 404: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 409: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation run x-summary-source: derived /v1/AgentEvaluation/{id}/evaluator-runs: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetEvaluatorRuns parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: type: array items: $ref: '#/components/schemas/EvaluatorRunResponse' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get evaluator runs x-summary-source: derived /v1/AgentEvaluation/{id}/evaluator-runs/comparison: get: tags: - AgentEvaluation operationId: AgentEvaluation_GetEvaluatorRunsComparison parameters: - name: id in: path required: true schema: type: string format: guid x-position: 1 - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/FoundryComparisonResponse' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation get evaluator runs comparison x-summary-source: derived /v1/AgentEvaluation/evaluation-limit: get: tags: - AgentEvaluation operationId: AgentEvaluation_CheckEvaluationLimit parameters: - name: x-correlation-id in: header schema: type: string responses: 200: description: '' content: application/json: schema: $ref: '#/components/schemas/EvaluationLimitResult' 401: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 403: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' 500: description: '' default: description: '' content: application/json: schema: $ref: '#/components/schemas/ProblemDetails' summary: Agent evaluation check evaluation limit x-summary-source: derived components: schemas: AgentDefinition: type: object additionalProperties: false required: - id - versionId - name - version properties: id: type: string format: guid versionId: type: string format: guid name: type: string version: type: string RunAgentEvaluationResponse: type: object additionalProperties: false properties: evaluatorRunId: type: string format: guid EvaluationType: type: string description: '' x-enumNames: - Standard - RedTeaming - Bias - StandardBiasDrift - Consistency enum: - Standard - RedTeaming - Bias - StandardBiasDrift - Consistency ConsistencyRunResult: type: object additionalProperties: false required: - actualOutput - accuracy properties: actualOutput: type: string truncated: type: boolean accuracy: type: number format: double detailedRationale: type: string latencyMs: type: - integer - 'null' format: int64 inputTokens: type: - integer - 'null' format: int32 outputTokens: type: - integer - 'null' format: int32 cost: type: - number - 'null' format: decimal AgentCapabilities: type: object additionalProperties: false properties: streaming: type: - boolean - 'null' pushNotifications: type: - boolean - 'null' extendedAgentCard: type: - boolean - 'null' extensions: type: - array - 'null' items: $ref: '#/components/schemas/AgentExtension' ScheduleModel: type: object additionalProperties: false properties: second: type: integer format: int32 maximum: 59 minimum: 0 minute: type: integer format: int32 maximum: 59 minimum: 0 hour: type: integer format: int32 maximum: 23 minimum: 0 executionHour: type: - integer - 'null' format: int32 maximum: 23 minimum: 0 day: type: integer format: int32 maximum: 31 minimum: 1 month: type: integer format: int32 maximum: 12 minimum: 1 scheduleType: $ref: '#/components/schemas/ScheduleType' daysOfWeek: type: - array - 'null' items: $ref: '#/components/schemas/DayOfWeek' monthlyWeekOccurrence: oneOf: - $ref: '#/components/schemas/MonthlyWeekOccurrence' timeZoneId: type: - string - 'null' maxLength: 100 minLength: 0 year: type: - integer - 'null' format: int32 maximum: 2099 minimum: 1970 startDate: type: - string - 'null' format: date-time endDate: type: - string - 'null' format: date-time ScheduleType: type: string description: '' x-enumNames: - Hourly - Daily - Weekly - Minute - OneTime - Monthly - Quarterly enum: - Hourly - Daily - Weekly - Minute - OneTime - Monthly - Quarterly JobStatus: type: string description: '' x-enumNames: - Creating - Created - Running - Cancelled - Success - Failed enum: - Creating - Created - Running - Cancelled - Success - Failed PagedResultOfComparisonResultRow: type: object additionalProperties: false properties: items: type: array items: $ref: '#/components/schemas/ComparisonResultRow' totalCount: type: integer format: int32 CreateEvaluationScheduleRequest: type: object additionalProperties: false required: - scheduleType - startDate properties: scheduleType: type: string minLength: 1 startDate: type: string format: date-time endDate: type: - string - 'null' format: date-time SecurityScheme: type: object discriminator: propertyName: type mapping: apiKey: '#/components/schemas/ApiKeySecurityScheme' http: '#/components/schemas/HttpSecurityScheme' oauth2: '#/components/schemas/OAuth2SecurityScheme' openIdConnect: '#/components/schemas/OpenIdConnectSecurityScheme' mutualTLS: '#/components/schemas/MutualTlsSecurityScheme' x-abstract: true additionalProperties: false required: - type properties: description: type: - string - 'null' type: type: string ThresholdBreachStatus: type: object additionalProperties: false properties: accuracy: oneOf: - $ref: '#/components/schemas/ThresholdBreachMetric' bias: oneOf: - $ref: '#/components/schemas/ThresholdBreachMetric' drift: oneOf: - $ref: '#/components/schemas/ThresholdBreachMetric' FoundryComparisonResponse: type: object additionalProperties: false properties: targets: type: array items: $ref: '#/components/schemas/AgentEvaluationTargetSummary' evaluators: type: array items: $ref: '#/components/schemas/FoundryComparisonEvaluator' rows: type: array items: $ref: '#/components/schemas/FoundryComparisonResultRow' aggregates: type: object additionalProperties: type: object additionalProperties: $ref: '#/components/schemas/FoundryComparisonScore' AgentEvaluationTargetSummary: type: object additionalProperties: false required: - id - kind - name properties: id: type: string format: guid kind: $ref: '#/components/schemas/CampaignTargetKind' pipelineVersionId: type: - string - 'null' format: guid name: type: string latestRunCompletedAt: type: - string - 'null' format: date-time WhatIfComparisonResult: type: object additionalProperties: false required: - comparisonSetId - baseline - variants properties: comparisonSetId: type: string format: guid baseline: $ref: '#/components/schemas/WhatIfVariantAggregate' variants: type: array items: $ref: '#/components/schemas/WhatIfVariantComparison' SingleEvaluationResult: type: object additionalProperties: false required: - agentVersionId - executionStatus - actualOutput - latencyMilliseconds - tokenCount - cost - accuracy properties: agentVersionId: type: string format: guid executionStatus: $ref: '#/components/schemas/JobTaskStatus' actualOutput: type: string latencyMilliseconds: type: integer format: int32 tokenCount: type: integer format: int32 cost: type: number format: decimal accuracy: type: number format: double detailedRationale: type: string SecurityRequirement: type: object additionalProperties: false properties: schemes: type: object additionalProperties: type: array items: type: string EvaluatorMetricResultResponse: type: object additionalProperties: false required: - id - source - evaluatorKey - score - scoreType properties: id: type: string format: guid source: $ref: '#/components/schemas/EvaluatorSource' evaluatorKey: type: string score: type: number format: double scoreType: $ref: '#/components/schemas/EvaluatorScoreType' reasonText: type: - string - 'null' PagedResultOfAgentEvaluationDto: type: object additionalProperties: false properties: items: type: array items: $ref: '#/components/schemas/AgentEvaluationDto' totalCount: type: integer format: int32 ModelIdentifierType: type: string description: '' x-enumNames: - None - Library - Project enum: - None - Library - Project DayOfWeek: type: string description: '' x-enumNames: - Sunday - Monday - Tuesday - Wednesday - Thursday - Friday - Saturday enum: - Sunday - Monday - Tuesday - Wednesday - Thursday - Friday - Saturday EvaluatorSource: type: string description: '' x-enumNames: - Airia - AzureFoundry enum: - Airia - AzureFoundry AgentEvaluationJobTaskResult: type: object additionalProperties: false required: - evaluationId - pipelineVersionId - datasetId - actualOutput - detailedRationale - accuracy properties: evaluationId: type: string format: guid pipelineVersionId: type: string format: guid datasetId: type: string format: guid actualOutput: type: string detailedRationale: type: string accuracy: type: number format: double isScored: type: boolean biasRiskScore: type: - number - 'null' format: double biasSeverity: type: - string - 'null' overallDriftScore: type: - number - 'null' format: double driftType: type: - string - 'null' judgeModelId: type: - string - 'null' format: guid judgeModelIdentifierType: oneOf: - $ref: '#/components/schemas/ModelIdentifierType' variantId: type: - string - 'null' format: guid isBaseline: type: boolean latencyMs: type: - integer - 'null' format: int64 inputTokens: type: - integer - 'null' format: int32 outputTokens: type: - integer - 'null' format: int32 estimatedCost: type: - number - 'null' format: decimal runStatus: type: - string - 'null' runLastError: type: - string - 'null' appliedOverrideSummary: type: - string - 'null' correlationId: type: - string - 'null' telemetryClassifier: type: - string - 'null' consistencyScore: type: - number - 'null' format: double consistencyRationale: type: - string - 'null' contradictions: type: - array - 'null' items: type: string outlierRunIndices: type: - array - 'null' items: type: integer format: int32 accuracyMin: type: - number - 'null' format: double accuracyMax: type: - number - 'null' format: double runs: type: - array - 'null' items: $ref: '#/components/schemas/ConsistencyRunResult' differencesSummary: type: - string - 'null' runExecutionIds: type: - array - 'null' items: type: string format: guid judgeInputTokens: type: - integer - 'null' format: int32 judgeOutputTokens: type: - integer - 'null' format: int32 judgeCost: type: - number - 'null' format: decimal judgeLatencyMs: type: - integer - 'null' format: int64 AgentEvaluationComparisonResponse: type: object additionalProperties: false required: - agents - evaluations properties: agents: type: array items: $ref: '#/components/schemas/AgentDefinition' evaluations: $ref: '#/components/schemas/PagedResultOfComparisonResultRow' AgentCardModel: type: object additionalProperties: false required: - projectId properties: projectId: type: string format: guid name: type: string description: type: string supportedInterfaces: type: array items: $ref: '#/components/schemas/AgentInterface' provider: oneOf: - $ref: '#/components/schemas/AgentProvider' version: type: string documentationUrl: type: - string - 'null' iconUrl: type: - string - 'null' capabilities: $ref: '#/components/schemas/AgentCapabilities' securitySchemes: type: - object - 'null' additionalProperties: $ref: '#/components/schemas/SecurityScheme' securityRequirements: type: - array - 'null' items: $ref: '#/components/schemas/SecurityRequirement' signatures: type: - array - 'null' items: $ref: '#/components/schemas/AgentCardSignature' defaultInputModes: type: - array - 'null' items: type: string defaultOutputModes: type: - array - 'null' items: type: string skills: type: array items: $ref: '#/components/schemas/AgentSkill' ToolPatchEntry: type: object additionalProperties: false properties: stepTitle: type: - string - 'null' toolInstanceId: type: - string - 'null' toolAggregateId: type: - string - 'null' configuration: oneOf: - $ref: '#/components/schemas/JsonNode' MonthlyWeekOccurrence: type: string description: '' x-enumNames: - First - Second - Third - Fourth - Last enum: - First - Second - Third - Fourth - Last JobTaskStatus: type: string description: '' x-enumNames: - Created - Ready - Scheduling - Scheduled - Cancelled - Running - Retrying - Success - Failed - Skipped enum: - Created - Ready - Scheduling - Scheduled - Cancelled - Running - Retrying - Success - Failed - Skipped FoundryComparisonScore: type: object additionalProperties: false required: - score - scoreType properties: score: type: number format: double scoreType: $ref: '#/components/schemas/EvaluatorScoreType' passed: type: - boolean - 'null' reasonText: type: - string - 'null' errorMessage: type: - string - 'null' EvaluationResultsType: type: string description: '' x-enumNames: - FullDetails - AccuracyBias - Drift enum: - FullDetails - AccuracyBias - Drift EvaluatorSelectionInput: type: object additionalProperties: false required: - evalSuiteEvaluatorId properties: evalSuiteEvaluatorId: type: string format: guid minLength: 1 initParamsJson: type: - string - 'null' AgentEvaluationAggregatedResultDto: type: object additionalProperties: false required: - agentVersionId - agentName - agentVersion - averageAccuracy - averageAgentExecutionDurationMs - averageModelCallCost - averageTokens - averageModelCallDurationMs properties: agentVersionId: type: string format: guid agentName: type: string agentVersion: type: string agentIcon: type: - string - 'null' agentIconId: type: - string - 'null' format: guid averageAccuracy: type: number format: double averageAgentExecutionDurationMs: type: integer format: int32 averageModelCallCost: type: number format: decimal averageTokens: type: number format: double averageModelCallDurationMs: type: integer format: int32 averageBias: type: - number - 'null' format: double averageDrift: type: - number - 'null' format: double averageConsistency: type: - number - 'null' format: double evaluatorCost: type: - number - 'null' format: decimal evaluatorTokens: type: - integer - 'null' format: int32 evaluatorLatencyMs: type: - integer - 'null' format: int64 runsPerTestCase: type: - integer - 'null' format: int32 thresholdBreach: oneOf: - $ref: '#/components/schemas/ThresholdBreachStatus' WhatIfMetricDelta: type: object additionalProperties: false required: - metricName - baselineValue - variantValue - delta - isImprovement properties: metricName: type: string baselineValue: type: number format: double variantValue: type: number format: double delta: type: number format: double isImprovement: type: boolean CampaignTargetKind: type: string description: '' x-enumNames: - PublishedPipelineVersion - ChangeScenario - ExternalHttp - SpmAgent - GatewayRoute enum: - PublishedPipelineVersion - ChangeScenario - ExternalHttp - SpmAgent - GatewayRoute UpsertEvaluationThresholdRequest: type: object additionalProperties: false properties: accuracyThreshold: type: - integer - 'null' format: int32 maximum: 100 minimum: 0 biasThreshold: type: - number - 'null' format: double maximum: 1 minimum: 0 driftThreshold: type: - integer - 'null' format: int32 maximum: 100 minimum: 0 ScenarioOverrides: type: object discriminator: propertyName: kind mapping: Internal: '#/components/schemas/InternalScenarioOverrides' External: '#/components/schemas/ExternalScenarioOverrides' x-abstract: true additionalProperties: false required: - kind properties: modelOverride: oneOf: - $ref: '#/components/schemas/ModelOverride' toolOverrides: oneOf: - $ref: '#/components/schemas/ToolPatch' kind: type: string PagedResultOfAgentEvaluationDetailedResultDto: type: object additionalProperties: false properties: items: type: array items: $ref: '#/components/schemas/AgentEvaluationDetailedResultDto' totalCount: type: integer format: int32 EvaluationConnectionStatus: type: string description: '' x-enumNames: - Active - DeletedPureFoundry - DeletedMixed enum: - Active - DeletedPureFoundry - DeletedMixed EvaluationTestCaseInput: type: object additionalProperties: false properties: input: type: string expectedOutput: type: string AgentEvaluationSummaryDto: type: object additionalProperties: false required: - latencyMs - tokenCount - cost - accuracy properties: latencyMs: type: number format: double tokenCount: type: integer format: int32 cost: type: number format: decimal accuracy: type: number format: double bias: type: - number - 'null' format: double drift: type: - number - 'null' format: double consistency: type: - number - 'null' format: double evaluatorCost: type: - number - 'null' format: decimal evaluatorTokens: type: - integer - 'null' format: int32 evaluatorLatencyMs: type: - number - 'null' format: double WhatIfVariantComparison: type: object additionalProperties: false required: - variant properties: variant: $ref: '#/components/schemas/WhatIfVariantAggregate' failed: type: boolean metricDeltas: type: array items: $ref: '#/components/schemas/WhatIfMetricDelta' latencyDeltaMs: type: - number - 'null' format: double estimatedCostDelta: type: - number - 'null' format: decimal totalTokensDelta: type: - number - 'null' format: double EvaluatorSelectionView: type: object additionalProperties: false required: - evaluatorKey properties: evalSuiteEvaluatorId: type: - string - 'null' format: guid source: oneOf: - $ref: '#/components/schemas/EvaluatorSource' evaluatorKey: type: string displayName: type: - string - 'null' deploymentName: type: - string - 'null' threshold: type: - number - 'null' format: double AiriaEvaluatorView: type: object additionalProperties: false required: - id - evaluatorKey - displayName - isEnabled - isDefault properties: id: type: string format: guid evaluatorKey: type: string displayName: type: string isEnabled: type: boolean isDefault: type: boolean JsonNode: type: object x-abstract: true additionalProperties: false properties: underlyingElement: {} options: oneOf: - $ref: '#/components/schemas/JsonNodeOptions' parent: oneOf: - $ref: '#/components/schemas/JsonNode' root: $ref: '#/components/schemas/JsonNode' AgentCardSignature: type: object additionalProperties: false properties: protected: type: string signature: type: string header: type: - object - 'null' additionalProperties: {} JsonNodeOptions: type: object additionalProperties: false properties: propertyNameCaseInsensitive: type: boolean FoundryComparisonEvaluator: type: object additionalProperties: false required: - key - scoreType properties: key: type: string scoreType: $ref: '#/components/schemas/EvaluatorScoreType' ConsistencyRunDto: type: object additionalProperties: false required: - actualOutput - accuracy properties: actualOutput: type: string truncated: type: boolean accuracy: type: number format: double latencyMs: type: - integer - 'null' format: int64 inputTokens: type: - integer - 'null' format: int32 outputTokens: type: - integer - 'null' format: int32 cost: type: - number - 'null' format: decimal AgentEvaluationTargetSummaryDto: type: object additionalProperties: false required: - completedCount - totalCount properties: completedCount: type: integer format: int32 totalCount: type: integer format: int32 ModelOverride: type: object additionalProperties: false properties: modelId: type: string identifierType: $ref: '#/components/schemas/ModelIdentifierType' AgentSkill: type: object additionalProperties: false properties: id: type: string name: type: string description: type: string tags: type: array items: type: string examples: type: - array - 'null' items: type: string inputModes: type: - array - 'null' items: type: string outputModes: type: - array - 'null' items: type: string securityRequirements: type: - array - 'null' items: $ref: '#/components/schemas/SecurityRequirement' AgentEvaluationDatasetDto: type: object additionalProperties: false required: - id - input - expectedOutput - context properties: id: type: string format: guid input: type: string expectedOutput: type: - string - 'null' context: type: - string - 'null' AgentProvider: type: object additionalProperties: false properties: organization: type: string url: type: string EvaluatorRunStatus: type: string description: '' x-enumNames: - Queued - InvokingTarget - Submitting - Polling - Succeeded - Failed - Cancelled enum: - Queued - InvokingTarget - Submitting - Polling - Succeeded - Failed - Cancelled AgentEvaluationDetailedResponse: type: object additionalProperties: false required: - agent - summary - evaluations properties: agent: $ref: '#/components/schemas/AgentDefinition' summary: $ref: '#/components/schemas/AgentEvaluationSummaryDto' evaluations: $ref: '#/components/schemas/PagedResultOfAgentEvaluationDetailedResultDto' FoundryComparisonResultRow: type: object additionalProperties: false required: - rowIndex properties: datasetItemId: type: - string - 'null' format: guid rowIndex: type: integer format: int32 input: type: - string - 'null' expectedOutput: type: - string - 'null' context: type: - string - 'null' cells: type: array items: $ref: '#/components/schemas/FoundryComparisonCell' AgentEvaluationVariant: type: object additionalProperties: false properties: variantId: type: string format: guid label: type: string isBaseline: type: boolean overrides: oneOf: - $ref: '#/components/schemas/ExternalScenarioOverrides' ThresholdBreachMetric: type: object additionalProperties: false required: - breached - actualValue - thresholdValue properties: breached: type: boolean actualValue: type: number format: double thresholdValue: type: number format: double EvaluationLimitResult: type: object additionalProperties: false properties: limitReached: type: boolean limit: type: integer format: int32 ScenarioOverrideKind: type: string description: '' x-enumNames: - Internal - External enum: - Internal - External UpdateEvaluationScheduleRequest: type: object additionalProperties: false required: - scheduleType - startDate properties: scheduleType: type: string minLength: 1 startDate: type: string format: date-time endDate: type: - string - 'null' format: date-time isCancelled: type: - boolean - 'null' AgentEvaluationAggregatedResponse: type: object additionalProperties: false required: - aggregatedEvaluations - evaluationId - evaluationName - evaluationDescription - runAt - evaluationCost - status properties: aggregatedEvaluations: type: array items: $ref: '#/components/schemas/AgentEvaluationAggregatedResultDto' evaluationId: type: string format: guid evaluationName: type: string evaluationDescription: type: - string - 'null' runAt: type: - string - 'null' format: date-time evaluationCost: type: number format: decimal status: $ref: '#/components/schemas/JobStatus' EvaluationThresholdResponse: type: object additionalProperties: false required: - evaluationId properties: evaluationId: type: string format: guid accuracyThreshold: type: - integer - 'null' format: int32 biasThreshold: type: - number - 'null' format: double driftThreshold: type: - integer - 'null' format: int32 AgentEvaluationDetailedResultDto: type: object additionalProperties: false required: - input - expectedOutput - executionStatus - actualOutput - latencyMs - tokenCount - cost - accuracy properties: input: type: string expectedOutput: type: - string - 'null' executionStatus: $ref: '#/components/schemas/JobTaskStatus' actualOutput: type: string latencyMs: type: integer format: int32 tokenCount: type: integer format: int32 cost: type: number format: decimal accuracy: type: number format: double detailedRationale: type: string bias: type: - number - 'null' format: double biasType: type: - string - 'null' drift: type: - number - 'null' format: double driftType: type: - string - 'null' consistencyScore: type: - number - 'null' format: double accuracyMin: type: - number - 'null' format: double accuracyMax: type: - number - 'null' format: double runCount: type: - integer - 'null' format: int32 consistencyRationale: type: - string - 'null' contradictions: type: - array - 'null' items: type: string outlierRunIndices: type: - array - 'null' items: type: integer format: int32 runs: type: - array - 'null' items: $ref: '#/components/schemas/ConsistencyRunDto' differencesSummary: type: - string - 'null' judgeTokens: type: - integer - 'null' format: int32 judgeCost: type: - number - 'null' format: decimal judgeLatencyMs: type: - integer - 'null' format: int64 ToolPatch: type: object additionalProperties: false properties: add: type: - array - 'null' items: $ref: '#/components/schemas/ToolPatchEntry' remove: type: - array - 'null' items: $ref: '#/components/schemas/ToolPatchEntry' modify: type: - array - 'null' items: $ref: '#/components/schemas/ToolPatchEntry' CreateAgentEvaluationResponse: type: object additionalProperties: false required: - id - evaluationName - agentVersionIds properties: id: type: string format: guid evaluationName: type: string description: type: - string - 'null' agentVersionIds: type: array items: type: string format: guid modelId: type: - string - 'null' format: guid datasetIds: type: - array - 'null' items: type: string format: guid schedule: oneOf: - $ref: '#/components/schemas/ScheduleModel' originId: type: - string - 'null' format: guid originSource: type: - string - 'null' originContext: type: - object - 'null' additionalProperties: type: string EvaluationScheduleResponse: type: object additionalProperties: false required: - scheduleId - scheduleType - startDate properties: scheduleId: type: string format: guid scheduleType: type: string startDate: type: string format: date-time endDate: type: - string - 'null' format: date-time nextEvaluationDate: type: - string - 'null' format: date-time isCancelled: type: boolean WhatIfBaselineMode: type: string description: '' x-enumNames: - FreshInJob - ReuseExisting enum: - FreshInJob - ReuseExisting FoundryComparisonCell: type: object additionalProperties: false required: - targetId properties: targetId: type: string format: guid response: type: - string - 'null' evaluatorScores: type: object additionalProperties: $ref: '#/components/schemas/FoundryComparisonScore' ProblemDetails: type: object additionalProperties: {} properties: type: type: - string - 'null' title: type: - string - 'null' status: type: - integer - 'null' format: int32 detail: type: - string - 'null' instance: type: - string - 'null' PagedResultOfAgentEvaluationDatasetDto: type: object additionalProperties: false properties: items: type: array items: $ref: '#/components/schemas/AgentEvaluationDatasetDto' totalCount: type: integer format: int32 AgentExtension: type: object additionalProperties: false properties: uri: type: string description: type: - string - 'null' required: type: - boolean - 'null' params: type: - object - 'null' additionalProperties: {} EvaluatorRunResponse: type: object additionalProperties: false required: - id - agentEvaluationId - source - status properties: id: type: string format: guid agentEvaluationId: type: string format: guid source: $ref: '#/components/schemas/EvaluatorSource' status: $ref: '#/components/schemas/EvaluatorRunStatus' providerRunId: type: - string - 'null' providerEvalId: type: - string - 'null' submittedAt: type: - string - 'null' format: date-time completedAt: type: - string - 'null' format: date-time attemptNumber: type: integer format: int32 createdAt: type: - string - 'null' format: date-time agentEvaluationTargetId: type: - string - 'null' format: guid target: oneOf: - $ref: '#/components/schemas/AgentEvaluationTargetSummary' metricResults: type: array items: $ref: '#/components/schemas/EvaluatorMetricResultResponse' GetAgentEvaluationResponse: type: object additionalProperties: false required: - id - evaluationName - agentVersionIds - datasetIds properties: id: type: string format: guid evaluationName: type: string description: type: - string - 'null' agentVersionIds: type: array items: type: string format: guid modelId: type: - string - 'null' format: guid datasetIds: type: array items: type: string format: guid evaluationType: $ref: '#/components/schemas/EvaluationType' runsPerTestCase: type: - integer - 'null' format: int32 modelIdentifierType: $ref: '#/components/schemas/ModelIdentifierType' externalAgentUrls: type: array items: type: string schedule: oneOf: - $ref: '#/components/schemas/ScheduleModel' baselineEvaluationId: type: - string - 'null' format: guid evaluatorKeys: type: array items: type: string evaluatorSelections: type: array items: $ref: '#/components/schemas/EvaluatorSelectionView' evaluatorSource: oneOf: - $ref: '#/components/schemas/EvaluatorSource' connectionStatus: $ref: '#/components/schemas/EvaluationConnectionStatus' spmAgentIds: type: array items: type: string format: guid variants: type: - array - 'null' items: $ref: '#/components/schemas/AgentEvaluationVariant' comparisonSetId: type: - string - 'null' format: guid baselineMode: $ref: '#/components/schemas/WhatIfBaselineMode' isWhatIfVariantSet: type: boolean whatIf: oneOf: - $ref: '#/components/schemas/WhatIfComparisonSection' ExternalScenarioOverrides: allOf: - $ref: '#/components/schemas/ScenarioOverrides' - type: object additionalProperties: false properties: overrideKind: $ref: '#/components/schemas/ScenarioOverrideKind' systemPromptOverride: type: - string - 'null' additionalInstructions: type: - string - 'null' configOverrides: type: - object - 'null' additionalProperties: $ref: '#/components/schemas/JsonNode' AgentEvaluationDto: type: object additionalProperties: false required: - evaluationId - evaluationJobId - name - status - agents - totalTasksCount - completedTasksCount - evaluationModelName - createdAt - createdBy properties: evaluationId: type: string format: guid evaluationJobId: type: string format: guid name: type: string status: $ref: '#/components/schemas/JobStatus' agents: type: array items: $ref: '#/components/schemas/AgentDefinition' totalTasksCount: type: integer format: int32 completedTasksCount: type: integer format: int32 evaluationModelName: type: string createdAt: type: - string - 'null' format: date-time createdBy: type: string ComparisonResultRow: type: object additionalProperties: false required: - datasetId - input - expectedOutput - comparisonResults properties: datasetId: type: string format: guid input: type: string expectedOutput: type: - string - 'null' comparisonResults: type: array items: $ref: '#/components/schemas/SingleEvaluationResult' WhatIfComparisonSection: type: object additionalProperties: false required: - comparisonSetId - variantResults properties: comparisonSetId: type: string format: guid variantResults: type: array items: $ref: '#/components/schemas/AgentEvaluationJobTaskResult' comparison: oneOf: - $ref: '#/components/schemas/WhatIfComparisonResult' WhatIfVariantAggregate: type: object additionalProperties: false properties: variantId: type: - string - 'null' format: guid isBaseline: type: boolean runCount: type: integer format: int32 metrics: type: object additionalProperties: type: number format: double averageLatencyMs: type: - number - 'null' format: double averageEstimatedCost: type: - number - 'null' format: decimal averageTotalTokens: type: - number - 'null' format: double AgentInterface: type: object additionalProperties: false properties: url: type: string protocolBinding: type: string protocolVersion: type: string tenant: type: - string - 'null' EvaluatorScoreType: type: string description: '' x-enumNames: - Scale1To5 - Scale1To7 - Boolean - Percentage - Decimal enum: - Scale1To5 - Scale1To7 - Boolean - Percentage - Decimal securitySchemes: ApiKey: type: apiKey description: API Key Authentication name: X-API-Key in: header Cookies: type: apiKey description: Cookie based Authentication name: Cookie in: header x-generator: NSwag v14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))