package llm import ( "errors" "os" "strings" "time" "github.com/yeomyeonggeori/blueclaw/internal/capability" "github.com/yeomyeonggeori/blueclaw/internal/config" "github.com/yeomyeonggeori/blueprotocol/model" "github.com/yeomyeonggeori/blueprotocol/model/decisions" "github.com/yeomyeonggeori/blueprotocol/model/openaicompatible" ) var ModelTiers = []string{"xlow", "low", "medium", "high", "xhigh", "max"} // TierProvider carries what the tier reaches alongside the provider that // reaches it, so a caller can see that two tiers land on the same model without // being told which model that is. type TierProvider struct { Provider LanguageModelProvider Reaches string } type TierProviderFactory func(modelTier string) (TierProvider, error) // NewTierProviderFactory answers with the provider a tier reaches its model // through. A document that names endpoints is a company plane reaching them // itself; a document that names the capability route is a device, where // capabilityd decides between a local model and a remote one. func NewTierProviderFactory(runtimeConfiguration config.RuntimeConfiguration) (TierProviderFactory, error) { languageModelConfiguration := runtimeConfiguration.LanguageModel hasEndpoints := len(languageModelConfiguration.Tiers) > 0 hasCapabilityRoute := capabilityRouteIsNamed(languageModelConfiguration.Capability) if hasEndpoints && hasCapabilityRoute { return nil, errors.New("the language model configuration names both tier endpoints and the capability route; it must name one") } if hasEndpoints { return endpointTierProviderFactory(languageModelConfiguration.Tiers), nil } if hasCapabilityRoute { return capabilityTierProviderFactory(runtimeConfiguration), nil } return nil, errors.New("the language model configuration names no tier endpoints and no capability route") } func endpointTierProviderFactory(tiers map[string][]config.ModelEndpointConfiguration) TierProviderFactory { return func(modelTier string) (TierProvider, error) { rungs := tiers[modelTier] if len(rungs) == 0 { return TierProvider{}, errors.New("the language model configuration gives the " + modelTier + " tier no endpoint") } var provider LanguageModelProvider reached := []string{} for rungIndex := len(rungs) - 1; rungIndex >= 0; rungIndex-- { rungProvider, errorValue := endpointProvider(rungs[rungIndex]) if errorValue != nil { return TierProvider{}, errorValue } reached = append([]string{rungs[rungIndex].Endpoint + " " + rungs[rungIndex].Model}, reached...) if provider == nil { provider = rungProvider continue } provider = FallbackLanguageModelProvider{ PrimaryProvider: rungProvider, FallbackProvider: provider, PrimaryLabel: modelTier, FallbackLabel: modelTier, FallbackServesTheSameTier: true, } } return TierProvider{Provider: provider, Reaches: strings.Join(reached, ", ")}, nil } } func endpointProvider(endpointConfiguration config.ModelEndpointConfiguration) (LanguageModelProvider, error) { apiKey, errorValue := endpointAPIKey(endpointConfiguration) if errorValue != nil { return nil, errorValue } provider, errorValue := openaicompatible.Endpoint{ URL: endpointConfiguration.Endpoint, ModelName: endpointConfiguration.Model, APIKey: apiKey, ProviderOrder: endpointConfiguration.ProviderOrder, ProviderSort: endpointConfiguration.ProviderSort, ReasoningEffort: endpointConfiguration.ReasoningEffort, }.Provider() if errorValue != nil { return nil, errorValue } if apiKeyPath := keyFilePath(endpointConfiguration); apiKeyPath != "" { provider.UseHTTPClient(keyFileClient(apiKeyPath, endpointConfiguration.RequesterEmailHeader)) } return provider, nil } func capabilityTierProviderFactory(runtimeConfiguration config.RuntimeConfiguration) TierProviderFactory { return func(modelTier string) (TierProvider, error) { modelName := capabilityTierModelName(runtimeConfiguration.LanguageModel.Capability, modelTier) if modelName == "" { return TierProvider{}, errors.New("the capability language model configuration gives the " + modelTier + " tier no model") } return TierProvider{ Provider: NewCapabilityLLMClientForTier(runtimeConfiguration, modelTier, modelName), Reaches: modelName, }, nil } } func capabilityRouteIsNamed(capabilityConfiguration config.LanguageModelCapabilityConfiguration) bool { for _, modelTier := range ModelTiers { if capabilityTierModelName(capabilityConfiguration, modelTier) != "" { return true } } return false } func capabilityTierModelName(capabilityConfiguration config.LanguageModelCapabilityConfiguration, modelTier string) string { switch modelTier { case "max": return strings.TrimSpace(capabilityConfiguration.MaxModel) case "xhigh": return strings.TrimSpace(capabilityConfiguration.XHighModel) case "high": return strings.TrimSpace(capabilityConfiguration.HighModel) case "medium": return strings.TrimSpace(capabilityConfiguration.MediumModel) case "low": return strings.TrimSpace(capabilityConfiguration.LowModel) case "xlow": return strings.TrimSpace(capabilityConfiguration.XLowModel) default: return "" } } func endpointAPIKey(endpointConfiguration config.ModelEndpointConfiguration) (string, error) { environmentName := strings.TrimSpace(endpointConfiguration.APIKeyEnvironment) if environmentName == "" { return readAPIKey(endpointConfiguration.APIKeyPath) } if strings.TrimSpace(endpointConfiguration.APIKeyPath) != "" { return "", errors.New("the model endpoint " + endpointConfiguration.Endpoint + " names both apiKeyPath and apiKeyEnvironment; it must name one") } apiKey := strings.TrimSpace(os.Getenv(environmentName)) if apiKey == "" { return "", errors.New("the model endpoint " + endpointConfiguration.Endpoint + " reads its key from " + environmentName + ", which is not set") } return apiKey, nil } func readAPIKey(apiKeyPath string) (string, error) { trimmedPath := strings.TrimSpace(apiKeyPath) if trimmedPath == "" { return "", nil } keyDocument, errorValue := os.ReadFile(trimmedPath) if errorValue != nil { return "", errorValue } apiKey := strings.TrimSpace(string(keyDocument)) if apiKey == "" { return "", errors.New("the model endpoint's api key file is empty: " + trimmedPath) } return apiKey, nil } type ConfiguredEmbedder interface { EmbeddingProvider EmbeddingModelName() string } func NewConfiguredEmbeddingProvider(runtimeConfiguration config.RuntimeConfiguration) (ConfiguredEmbedder, error) { embeddingConfiguration := runtimeConfiguration.LanguageModel.Embedding modelName := ConfiguredEmbeddingModelName(runtimeConfiguration) dimensions := ConfiguredEmbeddingDimensions(runtimeConfiguration) if strings.TrimSpace(embeddingConfiguration.Endpoint) == "" { return CapabilityEmbeddingClient{ CapabilityClient: newCapabilityClient(runtimeConfiguration), ModelName: modelName, ExecutionMode: firstNonEmpty(runtimeConfiguration.Memory.EmbeddingExecutionMode, runtimeConfiguration.LanguageModel.Capability.ExecutionMode), OutputDimensions: dimensions, }, nil } apiKey, errorValue := endpointAPIKey(embeddingConfiguration) if errorValue != nil { return nil, errorValue } provider, errorValue := openaicompatible.Endpoint{ URL: embeddingConfiguration.Endpoint, ModelName: modelName, APIKey: apiKey, }.EmbeddingProvider() if errorValue != nil { return nil, errorValue } if apiKeyPath := keyFilePath(embeddingConfiguration); apiKeyPath != "" { provider.UseHTTPClient(keyFileClient(apiKeyPath, embeddingConfiguration.RequesterEmailHeader)) } return fixedWidthEmbedder{provider: provider, dimensions: dimensions}, nil } func NewCapabilityLLMClientForTier(runtimeConfiguration config.RuntimeConfiguration, modelTier string, modelName string) CapabilityLLMClient { return CapabilityLLMClient{ CapabilityClient: newCapabilityClient(runtimeConfiguration), ModelName: strings.TrimSpace(modelName), ModelTier: strings.TrimSpace(modelTier), ExecutionMode: runtimeConfiguration.LanguageModel.Capability.ExecutionMode, } } func newCapabilityClient(runtimeConfiguration config.RuntimeConfiguration) capability.Client { return capability.NewClient(capability.Configuration{ Endpoint: runtimeConfiguration.Capabilities.Endpoint, UnixSocketPath: runtimeConfiguration.Capabilities.UnixSocketPath, Timeout: time.Duration(runtimeConfiguration.Capabilities.TimeoutSecond) * time.Second, }) } func NewConfiguredDecisionModel(runtimeConfiguration config.RuntimeConfiguration) (model.DecisionModel, error) { decisionConfiguration := runtimeConfiguration.LanguageModel.Decision if strings.TrimSpace(decisionConfiguration.Endpoint) != "" { return endpointDecisionModel(decisionConfiguration) } modelName := strings.TrimSpace(runtimeConfiguration.LanguageModel.Capability.DecisionModel) if modelName == "" { return nil, errors.New("the capability language model configuration names no decision model") } return CapabilityDecisionClient{CapabilityLLMClient: CapabilityLLMClient{ CapabilityClient: newCapabilityClient(runtimeConfiguration), ModelName: modelName, ExecutionMode: runtimeConfiguration.LanguageModel.Capability.ExecutionMode, }}, nil } func NewConfiguredVisualReviewModel(runtimeConfiguration config.RuntimeConfiguration) (model.DecisionModel, error) { visualReviewConfiguration := runtimeConfiguration.LanguageModel.VisualReview if strings.TrimSpace(visualReviewConfiguration.Endpoint) == "" { return nil, nil } return endpointDecisionModel(visualReviewConfiguration) } func endpointDecisionModel(decisionConfiguration config.ModelEndpointConfiguration) (model.DecisionModel, error) { modelName := strings.TrimSpace(decisionConfiguration.Model) if modelName == "" { return nil, errors.New("the decision endpoint " + decisionConfiguration.Endpoint + " names no model") } apiKey, errorValue := endpointAPIKey(decisionConfiguration) if errorValue != nil { return nil, errorValue } if apiKeyPath := keyFilePath(decisionConfiguration); apiKeyPath != "" { return keyFileDecisionModel{endpointURL: decisionConfiguration.Endpoint, modelName: modelName, apiKeyPath: apiKeyPath, requesterEmailHeader: decisionConfiguration.RequesterEmailHeader}, nil } return decisions.Endpoint{URL: decisionConfiguration.Endpoint, ModelName: modelName, APIKey: apiKey, HTTPClient: requesterNamingDecisionClient(decisionConfiguration.RequesterEmailHeader)}.DecisionModel(), nil }