//go:build !nobundledharness package app import ( "context" "encoding/json" "errors" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "sort" "strings" "testing" "time" "github.com/yeomyeonggeori/blueclaw/internal/agentruntime" "github.com/yeomyeonggeori/blueclaw/internal/capability" "github.com/yeomyeonggeori/blueclaw/internal/config" "github.com/yeomyeonggeori/blueclaw/internal/connectors" "github.com/yeomyeonggeori/blueclaw/internal/defaultharness" "github.com/yeomyeonggeori/blueclaw/internal/llm" "github.com/yeomyeonggeori/blueclaw/internal/protocolidentity" "github.com/yeomyeonggeori/blueclaw/internal/runtimecontrol" "github.com/yeomyeonggeori/blueclaw/internal/task" "github.com/yeomyeonggeori/blueprotocol/agentcontract" ) func TestTaskIntakeControllerStartsUnquiesced(t *testing.T) { controller := runtimecontrol.NewTaskIntakeController(nil) if controller.IsQuiesced() { t.Fatal("fresh task intake controller must start unquiesced") } } func TestCapabilityToolDescriptorsPreserveResultContracts(t *testing.T) { inputIntentSchema := json.RawMessage(`{"type":"object","properties":{"title":{"type":"string"}},"additionalProperties":false}`) descriptors := capabilityToolDescriptors([]config.CapabilityToolDescriptor{{ Name: "task_add", InputIntentSchema: inputIntentSchema, ResultContract: &config.CapabilityToolResultContract{ Schema: json.RawMessage(`{"type":"object","properties":{"taskID":{"type":"string"}},"required":["taskID"],"additionalProperties":false}`), Effects: []config.CapabilityResourceEffectContract{{ ObjectType: "task", Effect: "created", ResultField: "taskID", EffectIdentity: "id", }}, EvidenceCondition: &config.EvidenceCondition{ ResultField: "taskID", Equals: json.RawMessage(`"task-1"`), }, }, }}) if len(descriptors) != 1 || descriptors[0].ResultContract == nil { t.Fatalf("expected mapped result contract, got %+v", descriptors) } if string(descriptors[0].InputIntentSchema) != string(inputIntentSchema) { t.Fatalf("expected mapped input intent schema, got %s", descriptors[0].InputIntentSchema) } if len(descriptors[0].ResultContract.Effects) != 1 || descriptors[0].ResultContract.Effects[0].ObjectType != "task" || descriptors[0].ResultContract.Effects[0].Effect != "created" || descriptors[0].ResultContract.Effects[0].ResultField != "taskID" || descriptors[0].ResultContract.Effects[0].EffectIdentity != "id" { t.Fatalf("expected mapped resource effect, got %+v", descriptors[0].ResultContract) } if descriptors[0].ResultContract.EvidenceCondition == nil || string(descriptors[0].ResultContract.EvidenceCondition.Equals) != `"task-1"` { t.Fatalf("expected mapped evidence condition, got %+v", descriptors[0].ResultContract) } } func TestResolveIntakeLanguageModelProviderUsesReliableTaskTierModel(t *testing.T) { runtimeConfiguration := configuredModelTierRuntime("") runtimeConfiguration.Agent.Intake.Enabled = true runtimeConfiguration.Agent.Intake.ExecutionMode = "auto" languageModelProvider, errorValue := resolveIntakeLanguageModelProvider(runtimeConfiguration, nil) if errorValue != nil { t.Fatal(errorValue) } fallbackLanguageModelProvider, isFallbackProvider := languageModelProvider.(llm.FallbackLanguageModelProvider) if !isFallbackProvider { t.Fatalf("expected fallback intake provider, got %T", languageModelProvider) } primaryClient, isPrimaryCapabilityClient := unwrapModelTier(fallbackLanguageModelProvider.PrimaryProvider).(llm.CapabilityLLMClient) if !isPrimaryCapabilityClient { t.Fatalf("expected primary capability intake provider, got %T", fallbackLanguageModelProvider.PrimaryProvider) } fallbackClient, isFallbackCapabilityClient := unwrapModelTier(fallbackLanguageModelProvider.FallbackProvider).(llm.CapabilityLLMClient) if !isFallbackCapabilityClient { t.Fatalf("expected fallback capability intake provider, got %T", fallbackLanguageModelProvider.FallbackProvider) } if primaryClient.ModelName != "vendor/medium" { t.Fatalf("expected the medium tier model for intake, got %q", primaryClient.ModelName) } if fallbackClient.ModelName != "vendor/high" { t.Fatalf("expected the high tier model as the intake fallback, got %q", fallbackClient.ModelName) } if primaryClient.ExecutionMode != "auto" || fallbackClient.ExecutionMode != "auto" { t.Fatalf("expected automatic intake execution mode, got %q and %q", primaryClient.ExecutionMode, fallbackClient.ExecutionMode) } } func TestMaximumXLowTierCapsTaskModelsAndUsesLowForImages(t *testing.T) { runtimeConfiguration := configuredModelTierRuntime("xlow") providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler)) if errorValue != nil { t.Fatal(errorValue) } for providerName, provider := range map[string]llm.LanguageModelProvider{ "low": providers.Low, "xlow": providers.XLow, "medium": providers.Medium, "high": providers.High, "xhigh": providers.XHigh, "max": providers.Max, } { modelNames := languageModelProviderNames(provider) if !reflect.DeepEqual(modelNames, []string{"vendor/low", "vendor/xlow"}) { t.Fatalf("expected %s provider to use only xlow text and low vision models, got %v", providerName, modelNames) } } } func TestMaximumLowTierCapsIntakeFallbacks(t *testing.T) { runtimeConfiguration := configuredModelTierRuntime("low") runtimeConfiguration.Agent.Intake.Enabled = true provider, errorValue := resolveIntakeLanguageModelProvider(runtimeConfiguration, nil) if errorValue != nil { t.Fatal(errorValue) } modelNames := languageModelProviderNames(provider) if !reflect.DeepEqual(modelNames, []string{"vendor/low", "vendor/xlow"}) { t.Fatalf("expected intake to stay at or below low, got %v", modelNames) } } func configuredModelTierRuntime(maximumModelTier string) config.RuntimeConfiguration { runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.LanguageModel.MaximumModelTier = maximumModelTier runtimeConfiguration.LanguageModel.Capability.MaxModel = "vendor/max" runtimeConfiguration.LanguageModel.Capability.XHighModel = "vendor/xhigh" runtimeConfiguration.LanguageModel.Capability.HighModel = "vendor/high" runtimeConfiguration.LanguageModel.Capability.MediumModel = "vendor/medium" runtimeConfiguration.LanguageModel.Capability.LowModel = "vendor/low" runtimeConfiguration.LanguageModel.Capability.XLowModel = "vendor/xlow" return runtimeConfiguration } func languageModelProviderNames(provider llm.LanguageModelProvider) []string { modelNames := map[string]bool{} collectLanguageModelProviderNames(provider, modelNames) result := make([]string, 0, len(modelNames)) for modelName := range modelNames { result = append(result, modelName) } sort.Strings(result) return result } func collectLanguageModelProviderNames(provider llm.LanguageModelProvider, modelNames map[string]bool) { if tieredProvider, isTieredProvider := provider.(interface { UnderlyingProvider() llm.LanguageModelProvider }); isTieredProvider { collectLanguageModelProviderNames(tieredProvider.UnderlyingProvider(), modelNames) return } switch typedProvider := provider.(type) { case llm.CapabilityLLMClient: modelNames[typedProvider.ModelName] = true case llm.FallbackLanguageModelProvider: collectLanguageModelProviderNames(typedProvider.PrimaryProvider, modelNames) collectLanguageModelProviderNames(typedProvider.FallbackProvider, modelNames) case llm.VisionFallbackProvider: collectLanguageModelProviderNames(typedProvider.TextOnlyModel, modelNames) collectLanguageModelProviderNames(typedProvider.VisionModel, modelNames) } } func unwrapModelTier(provider llm.LanguageModelProvider) llm.LanguageModelProvider { if tieredProvider, isTieredProvider := provider.(interface { UnderlyingProvider() llm.LanguageModelProvider }); isTieredProvider { return tieredProvider.UnderlyingProvider() } return provider } func TestLoadAgentInstructionPromptUsesIdentitySoulAgentsAndSkills(t *testing.T) { workspacePath := t.TempDir() skillDirectoryPath := filepath.Join(workspacePath, ".agents", "skills", "browser") if errorValue := os.MkdirAll(skillDirectoryPath, 0o755); errorValue != nil { t.Fatalf("expected skill directory: %v", errorValue) } if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴", "Intern Kim"], "handle": "internkim", "role": "the company's AI intern"}`), 0o600); errorValue != nil { t.Fatalf("expected identity file: %v", errorValue) } if errorValue := os.WriteFile(filepath.Join(workspacePath, "soul.json"), []byte(`{"schemaVersion": 1, "values": ["Lead with the result."]}`), 0o600); errorValue != nil { t.Fatalf("expected soul file: %v", errorValue) } if errorValue := os.WriteFile(filepath.Join(workspacePath, "AGENTS.md"), []byte("Use agent-browser for web automation."), 0o600); errorValue != nil { t.Fatalf("expected agents file: %v", errorValue) } if errorValue := os.WriteFile(filepath.Join(skillDirectoryPath, "SKILL.md"), []byte("Run agent-browser snapshot -i after navigation."), 0o600); errorValue != nil { t.Fatalf("expected skill file: %v", errorValue) } runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath instructionBundle := loadAgentInstructionBundle(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil)) for _, expectedFragment := range []string{"Your name is 김인턴.", "\"Intern Kim\"", "@internkim", "Your role: the company's AI intern", "What you hold to:\n- Lead with the result.", "Use agent-browser for web automation."} { if !strings.Contains(instructionBundle.Prompt, expectedFragment) { t.Fatalf("expected instruction prompt to contain %q, got %q", expectedFragment, instructionBundle.Prompt) } } if len(instructionBundle.Skills) == 0 || instructionBundle.Skills[0].Name != "browser" { t.Fatalf("expected skill metadata to be loaded: %+v", instructionBundle.Skills) } } func TestLoadAgentIdentityReadsTheIdentityDocument(t *testing.T) { workspacePath := t.TempDir() if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴"], "handle": "internkim"}`), 0o600); errorValue != nil { t.Fatalf("expected identity file: %v", errorValue) } runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath agentIdentity := loadAgentIdentity(runtimeConfiguration) if agentIdentity.Name != "김인턴" || agentIdentity.Handle != "internkim" { t.Fatalf("expected the identity document's name and handle, got %+v", agentIdentity) } } func TestAPersonaDocumentTheSchemaRefusesIsReportedAndLeftOut(t *testing.T) { workspacePath := t.TempDir() if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴"], "handle": "internkim", "nickname": "kim"}`), 0o600); errorValue != nil { t.Fatalf("expected identity file: %v", errorValue) } runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath instructions := loadAgentInstructions(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil)) if strings.Contains(instructions.Bundle.Prompt, "김인턴") { t.Fatalf("expected the refused identity to stay out of the prompt, got %q", instructions.Bundle.Prompt) } if len(instructions.RejectedDocuments) != 1 || !strings.Contains(instructions.RejectedDocuments[0].Error.Error(), "nickname") { t.Fatalf("expected the refusal to be reported with its reason, got %+v", instructions.RejectedDocuments) } if agentIdentity := loadAgentIdentity(runtimeConfiguration); agentIdentity.Name != "" { t.Fatalf("expected no identity from a refused document, got %+v", agentIdentity) } } func TestLoadAgentInstructionBundleDiscoversAddedUserSkill(t *testing.T) { workspacePath := t.TempDir() skillDirectoryPath := filepath.Join(workspacePath, ".agents", "skills", "research-helper") if errorValue := os.MkdirAll(skillDirectoryPath, 0o755); errorValue != nil { t.Fatalf("expected skill directory: %v", errorValue) } skillDocument := `--- name: research-helper description: Help with research tasks and source lookup requests. --- Research helper body. ` if errorValue := os.WriteFile(filepath.Join(skillDirectoryPath, "SKILL.md"), []byte(skillDocument), 0o600); errorValue != nil { t.Fatalf("expected skill file: %v", errorValue) } runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath instructionBundle := loadAgentInstructionBundle(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil)) if len(instructionBundle.Skills) != 1 || instructionBundle.Skills[0].Name != "research-helper" { t.Fatalf("expected added user skill to be discovered, got %+v", instructionBundle.Skills) } if instructionBundle.Skills[0].Description != "Help with research tasks and source lookup requests." { t.Fatalf("expected standard skill fields, got %+v", instructionBundle.Skills[0]) } } func TestDeriveAllowedToolNamesByProfileKeepsDomainCapabilitiesOutOfBaseline(t *testing.T) { runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.AgentProfiles = []config.AgentProfileConfiguration{ {Name: "default", AllowedToolNames: []string{"bash"}}, } runtimeConfiguration.Capabilities.ToolDescriptors = []config.CapabilityToolDescriptor{{Name: "event_add"}} allowedToolNamesByProfile := deriveAllowedToolNamesByProfile(runtimeConfiguration) defaultProfileToolNames := allowedToolNamesByProfile["default"] if containsString(defaultProfileToolNames, "event_add") { t.Fatalf("expected domain capability to stay out of profile baseline, got %+v", defaultProfileToolNames) } for _, expectedToolName := range []string{"bash", "file_deliver", "skill_search", "file_read", "write", "edit", "file_preview", "image_read"} { if !containsString(defaultProfileToolNames, expectedToolName) { t.Fatalf("expected baseline tool %q, got %+v", expectedToolName, defaultProfileToolNames) } } if containsString(defaultProfileToolNames, "ask_confirm") { t.Fatalf("expected runtime-owned confirmation to stay out of model tools, got %+v", defaultProfileToolNames) } if containsString(defaultProfileToolNames, "ask_input") { t.Fatalf("expected typed user input to stay out of the baseline tools, got %+v", defaultProfileToolNames) } } func TestApplicationChecksProtocolIdentityOnceAndStoresResult(t *testing.T) { protocolVersion := "0.4.0" aggregateProtocolHash := "58ff1977989bacbf2db3fdce08fd57c9b52f344ca747a3322f4e60bdf6052a78" requestCount := 0 server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { requestCount++ responseWriter.Header().Set("Content-Type", "application/json") _, _ = responseWriter.Write([]byte(`{"status":"ok","protocolVersion":"` + protocolVersion + `","aggregateProtocolHash":"` + aggregateProtocolHash + `"}`)) })) defer server.Close() runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Logging.DirectoryPath = t.TempDir() runtimeConfiguration.Capabilities.Endpoint = server.URL runtimeConfiguration.Capabilities.ProtocolVersion = protocolVersion runtimeConfiguration.Capabilities.AggregateProtocolHash = aggregateProtocolHash application := NewApplication(runtimeConfiguration, "", InboundOptions{}, WithBundledACPFactory(defaultharness.NewFactory)) application.protocolIdentityExpected = protocolidentity.Identity{ ProtocolVersion: protocolVersion, AggregateProtocolHash: aggregateProtocolHash, } application.protocolIdentityChecker = protocolidentity.NewChecker(protocolidentity.Configuration{ CapabilityEndpoint: server.URL, HTTPClient: server.Client(), }) if errorValue := application.checkProtocolIdentity(); errorValue != nil { t.Fatalf("expected protocol identity check to pass: %v", errorValue) } if errorValue := application.checkProtocolIdentity(); errorValue != nil { t.Fatalf("expected repeated protocol identity check to reuse result: %v", errorValue) } if requestCount != 2 { t.Fatalf("expected one registry warm-up and one capabilityd request, got %d", requestCount) } if !application.protocolIdentityStatus.Passed { t.Fatalf("expected stored protocol identity result to pass: %+v", application.protocolIdentityStatus) } } func TestApplicationAutoResumeLaunchesAtMostFiveInterruptedTaskRuns(t *testing.T) { taskEventService := task.NewTaskEventService() taskRunService := task.NewTaskRunService(taskEventService) now := time.Now() repository := &applicationAutoResumeRepository{} for index := 0; index < 7; index++ { taskRun := task.TaskRun{ TaskRunID: "task-" + string(rune('a'+index)), RequesterPersonID: "person-1", OriginConversationID: "conversation-1", OriginReplyTargetID: "reply-1", Status: task.TaskStatusInterrupted, Prompt: "finish task", CreatedAt: now.Add(time.Duration(index) * time.Minute), UpdatedAt: now.Add(time.Duration(index) * time.Minute), } repository.taskRuns = append(repository.taskRuns, taskRun) taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskInterrupted, "runtime restarted") } taskRunService.UseRepository(repository) resumer := &applicationAutoResumeResumer{} application := &Application{ taskRunService: taskRunService, interruptedTaskResumer: resumer, interruptedTaskResumeDelay: 0, } application.resumeInterruptedTaskRuns(context.Background(), now.Add(time.Hour)) if len(resumer.taskRunIDs) != 5 { t.Fatalf("resume count = %d, want 5", len(resumer.taskRunIDs)) } for _, taskRunID := range resumer.taskRunIDs { if !taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRunID), "task.auto_resume_attempted") { t.Fatalf("expected auto-resume attempt event for %s", taskRunID) } } skippedCount := 0 for _, taskRun := range repository.taskRuns { if taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRun.TaskRunID), "task.auto_resume_skipped") { skippedCount++ } } if skippedCount != 2 { t.Fatalf("skipped count = %d, want 2", skippedCount) } } type applicationAutoResumeResumer struct { taskRunIDs []string failedTaskRunIDs []string } func (resumer *applicationAutoResumeResumer) CanResumeInterruptedTaskRun(task.TaskRun) bool { return true } func (resumer *applicationAutoResumeResumer) ResumeInterruptedTaskRun(_ context.Context, taskRun task.TaskRun) (connectors.ConnectorRuntimeResult, error) { resumer.taskRunIDs = append(resumer.taskRunIDs, taskRun.TaskRunID) return connectors.ConnectorRuntimeResult{Handled: true, TaskRunID: taskRun.TaskRunID}, nil } func (resumer *applicationAutoResumeResumer) FailUnresumedInterruptedTaskRun(_ context.Context, taskRun task.TaskRun, _ string) bool { resumer.failedTaskRunIDs = append(resumer.failedTaskRunIDs, taskRun.TaskRunID) return true } func TestApplicationFailsUnresumedInterruptedTaskRunsWithNotice(t *testing.T) { taskEventService := task.NewTaskEventService() taskRunService := task.NewTaskRunService(taskEventService) now := time.Now() repository := &applicationAutoResumeRepository{} for index := 0; index < 2; index++ { taskRun := task.TaskRun{ TaskRunID: "exhausted-" + string(rune('a'+index)), RequesterPersonID: "person-1", OriginConversationID: "conversation-1", OriginReplyTargetID: "reply-1", Status: task.TaskStatusInterrupted, FailureReason: task.TaskInterruptReasonRuntimeRestart, Prompt: "finish task", CreatedAt: now, UpdatedAt: now, } repository.taskRuns = append(repository.taskRuns, taskRun) taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskInterrupted, task.TaskInterruptReasonRuntimeRestart) taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskAutoResumeAttempted, `{"attemptCount":1}`) } taskRunService.UseRepository(repository) resumer := &applicationAutoResumeResumer{} application := &Application{ taskRunService: taskRunService, interruptedTaskResumer: resumer, interruptedTaskResumeDelay: 0, } application.resumeInterruptedTaskRuns(context.Background(), now.Add(time.Hour)) if len(resumer.taskRunIDs) != 0 { t.Fatalf("resume count = %d, want 0 for attempt-exhausted crash interruptions", len(resumer.taskRunIDs)) } if len(resumer.failedTaskRunIDs) != 2 { t.Fatalf("failed count = %d, want 2", len(resumer.failedTaskRunIDs)) } for _, taskRun := range repository.taskRuns { if !taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRun.TaskRunID), "task.auto_resume_abandoned") { t.Fatalf("expected auto-resume abandoned event for %s", taskRun.TaskRunID) } } } type applicationAutoResumeRepository struct { taskRuns []task.TaskRun } func (repository *applicationAutoResumeRepository) SaveTaskRun(taskRun task.TaskRun) error { repository.taskRuns = append(repository.taskRuns, taskRun) return nil } func (repository *applicationAutoResumeRepository) StartTaskRunAttempt(task.TaskRun, task.TaskAttempt) error { return nil } func (repository *applicationAutoResumeRepository) FinishTaskRunAttempt(task.TaskRun, task.TaskAttempt) error { return nil } func (repository *applicationAutoResumeRepository) TransitionTaskRun(transition task.TaskRunTransition) (task.TaskRun, error) { for index, taskRun := range repository.taskRuns { if taskRun.TaskRunID != transition.TaskRunID { continue } if !applicationTaskRunStatusAllowed(taskRun.Status, transition.FromStates) { return task.TaskRun{}, task.ErrIllegalTransition{ TaskRunID: transition.TaskRunID, CurrentStatus: taskRun.Status, FromStates: append([]task.TaskStatus{}, transition.FromStates...), ToState: transition.ToState, } } taskRun.Status = transition.ToState taskRun.UpdatedAt = transition.UpdatedAt repository.taskRuns[index] = taskRun return taskRun, nil } return task.TaskRun{}, errors.New("task run not found") } func applicationTaskRunStatusAllowed(status task.TaskStatus, allowedStatuses []task.TaskStatus) bool { for _, allowedStatus := range allowedStatuses { if status == allowedStatus { return true } } return false } func (repository *applicationAutoResumeRepository) FindTaskRun(taskRunID string) (task.TaskRun, bool, error) { for _, taskRun := range repository.taskRuns { if taskRun.TaskRunID == taskRunID { return taskRun, true, nil } } return task.TaskRun{}, false, nil } func (repository *applicationAutoResumeRepository) FindTaskAttempt(string) (task.TaskAttempt, bool, error) { return task.TaskAttempt{}, false, nil } func (repository *applicationAutoResumeRepository) ListTaskRun() ([]task.TaskRun, error) { return append([]task.TaskRun{}, repository.taskRuns...), nil } func (repository *applicationAutoResumeRepository) ListTaskRunByPersonID(personID string) ([]task.TaskRun, error) { taskRuns := []task.TaskRun{} for _, taskRun := range repository.taskRuns { if taskRun.RequesterPersonID == personID { taskRuns = append(taskRuns, taskRun) } } return taskRuns, nil } func (repository *applicationAutoResumeRepository) DeleteTaskRun(string, []string) (bool, error) { return false, nil } func (repository *applicationAutoResumeRepository) DeleteTaskRunsBefore(time.Time, []string) ([]string, error) { return nil, nil } func taskEventsContainApplicationEvent(taskEvents []task.TaskEvent, name string) bool { for _, taskEvent := range taskEvents { if taskEvent.Name == name { return true } } return false } func TestResolveTaskTierLanguageModelProvidersEscalateLowToMedium(t *testing.T) { runtimeConfiguration := configuredModelTierRuntime("") providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler)) if errorValue != nil { t.Fatal(errorValue) } lowProvider, isFallbackProvider := providers.Low.(llm.FallbackLanguageModelProvider) if !isFallbackProvider { t.Fatalf("expected low tier to escalate on failure, got %T", providers.Low) } if lowProvider.PrimaryLabel != "low" || lowProvider.FallbackLabel != "medium" { t.Fatalf("expected low/medium labels, got %q/%q", lowProvider.PrimaryLabel, lowProvider.FallbackLabel) } xLowProvider, isFallbackProvider := providers.XLow.(llm.FallbackLanguageModelProvider) if !isFallbackProvider { t.Fatalf("expected xlow tier fallback provider, got %T", providers.XLow) } if _, xLowClimbsTheLadder := xLowProvider.FallbackProvider.(llm.FallbackLanguageModelProvider); !xLowClimbsTheLadder { t.Fatalf("expected xlow to fall into the low-to-medium ladder, got %T", xLowProvider.FallbackProvider) } mediumProvider, isFallbackProvider := providers.Medium.(llm.FallbackLanguageModelProvider) if !isFallbackProvider { t.Fatalf("expected medium tier fallback provider, got %T", providers.Medium) } if _, mediumWouldRetryItself := mediumProvider.FallbackProvider.(llm.FallbackLanguageModelProvider); mediumWouldRetryItself { t.Fatalf("expected medium to fall back to the bare low model without climbing back up, got %T", mediumProvider.FallbackProvider) } } func TestResolveTaskTierLanguageModelProvidersKeepsBareLowWhenMediumMatchesLow(t *testing.T) { runtimeConfiguration := configuredModelTierRuntime("") runtimeConfiguration.LanguageModel.Capability.LowModel = "vendor/shared" runtimeConfiguration.LanguageModel.Capability.MediumModel = "vendor/shared" providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler)) if errorValue != nil { t.Fatal(errorValue) } if _, isFallbackProvider := providers.Low.(llm.FallbackLanguageModelProvider); isFallbackProvider { t.Fatal("expected bare low provider when the medium model equals the low model") } } func TestEveryTaskTierPostsTheModelItsEndpointWasGiven(t *testing.T) { receivedModelNames := make(chan string, 6) server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { requestDocument := struct { Model string `json:"model"` }{} if errorValue := json.NewDecoder(request.Body).Decode(&requestDocument); errorValue != nil { t.Errorf("expected a decodable completion request: %v", errorValue) } receivedModelNames <- requestDocument.Model responseWriter.Header().Set("Content-Type", "application/json") _, _ = responseWriter.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"call-1","type":"function","function":{"name":"answer","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}`)) })) defer server.Close() runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.LanguageModel.Tiers = map[string][]config.ModelEndpointConfiguration{} for _, modelTier := range llm.ModelTiers { runtimeConfiguration.LanguageModel.Tiers[modelTier] = []config.ModelEndpointConfiguration{ {Endpoint: server.URL, Model: "vendor/one"}, } } providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler)) if errorValue != nil { t.Fatal(errorValue) } for tierName, provider := range map[string]llm.LanguageModelProvider{ "xlow": providers.XLow, "low": providers.Low, "medium": providers.Medium, "high": providers.High, "xhigh": providers.XHigh, "max": providers.Max, } { if provider == nil { t.Fatalf("expected a %s tier provider", tierName) } request := llm.StructuredResponseRequest{ Messages: []llm.Message{{Role: "user", Content: "hello"}}, StructuredOutputSchema: llm.StructuredOutputSchema{ Name: "answer", Document: `{"type":"object","properties":{},"additionalProperties":false}`, }, } if _, errorValue := provider.GenerateStructuredResponse(context.Background(), request); errorValue != nil { t.Fatalf("expected the %s tier to reach the endpoint: %v", tierName, errorValue) } receivedModelName := <-receivedModelNames if receivedModelName != "vendor/one" { t.Fatalf("the %s tier asked the endpoint for %q, which it does not serve", tierName, receivedModelName) } } } func TestApplicationServesHealthWhenProtocolIdentityDisagrees(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) { responseWriter.Header().Set("Content-Type", "application/json") _, _ = responseWriter.Write([]byte(`{"status":"ok","protocolVersion":"0.4.0","aggregateProtocolHash":"b4b5c630888e6de30e52ec88809c9356cf2fb42d7e1189215d3de0d44e60c775"}`)) })) defer server.Close() runtimeConfiguration := config.RuntimeConfiguration{} runtimeConfiguration.Logging.DirectoryPath = t.TempDir() runtimeConfiguration.Capabilities.Endpoint = server.URL application := NewApplication(runtimeConfiguration, "", InboundOptions{}, WithBundledACPFactory(defaultharness.NewFactory)) application.httpServer.Addr = "127.0.0.1:0" application.protocolIdentityExpected = protocolidentity.Identity{ ProtocolVersion: "0.4.0", AggregateProtocolHash: "fccec45c4b3fc539159b3a293d61275ed2fc4ae738f371ec9122c1546b32a42f", } application.protocolIdentityChecker = protocolidentity.NewChecker(protocolidentity.Configuration{ CapabilityEndpoint: server.URL, HTTPClient: server.Client(), }) serveError := make(chan error, 1) go func() { serveError <- application.Start() }() t.Cleanup(func() { _ = application.httpServer.Close() }) select { case errorValue := <-serveError: t.Fatalf("a rejected protocol identity must not end the process: %v", errorValue) case <-time.After(500 * time.Millisecond): } if application.protocolIdentityStatus.Passed { t.Fatal("expected the stored protocol identity result to record the disagreement") } }