// Copyright 2026 TesterArmy. // Modified by Dario Kondratiuk. // SPDX-License-Identifier: Apache-2.0 using System.Collections.ObjectModel; using System.Globalization; using System.Text.Json; using E2E.Internal; namespace E2E; /// /// The resolved e2e.config.json. The JSON shape follows upstream e2e.config.ts: /// targets[].app.url, named agents, top-level timeouts in milliseconds, /// retries, cache, and secrets. Unknown keys fail with /// INVALID_CONFIG. E2ETest finds and applies this file; a fixture can /// still override each value. There is no default model. /// public sealed class E2EConfig { /// The file name looks for. public const string FileName = "e2e.config.json"; private static readonly Dictionary MovedKeys = new(StringComparer.Ordinal) { ["app"] = "the app under test is declared on its target: targets: [{ \"app\": { \"url\": ... } }]", ["engine"] = "the engine is chosen by the fixture; declare the target as targets: [{ \"platform\": \"web\" }]", ["agent"] = "agents are named: agents: { \"default\": { \"model\": ... } }", ["timeouts"] = "timeouts are top-level milliseconds: timeout, launchTimeout, actionTimeout, assertionTimeout, cleanupTimeout", }; // Upstream's REMOVED_TOP_LEVEL_KEYS: keys the runner used to accept, each mapped to what replaces it. private static readonly Dictionary RemovedTopLevelKeys = new(StringComparer.Ordinal) { ["specVersion"] = "delete it; the runner version is the format version", ["limits"] = "set maxInputTokens on each agent (it was limits.maxModelTokensPerCall); the runner fixes maxAgentContextBytes, maxLedgerBytes, and maxEventsPerStep", }; // Upstream's REMOVED_AGENT_KEYS: agent keys the runner used to accept, each mapped to what replaces it. private static readonly Dictionary RemovedAgentKeys = new(StringComparer.Ordinal) { ["timeout"] = "use judgmentTimeout, the deadline of one assert, waitFor, or extract call", ["maxTurns"] = "use maxModelCalls, the model requests one agent call may make", ["maxModelTokensPerCall"] = "use maxInputTokens", ["limits"] = "set maxInputTokens on the agent; the runner fixes the other limits", }; private static readonly string[] TopLevelKeys = [ "targets", "timeout", "launchTimeout", "actionTimeout", "assertionTimeout", "cleanupTimeout", "retries", "agents", "cache", "secrets", "trace", "video", ]; // Upstream keys the .NET port does not implement. They fail loudly rather than being ignored. private static readonly string[] UnsupportedTopLevelKeys = [ "projectId", "tests", "failOnSkippedFailure", "workers", "artifacts", "output", "reporters", "credentials", ]; private static readonly string[] TargetKeys = ["name", "platform", "app", "trace", "video"]; private static readonly string[] UnsupportedTargetKeys = ["engine"]; private static readonly string[] AppKeys = ["url", "command", "readyUrl", "identity", "environment"]; private static readonly string[] MobileAppKeys = ["bundleId", "appPath", "command", "readyUrl", "identity", "environment", "launchArguments", "permissions"]; private static readonly string[] WindowsAppKeys = ["bundleId", "appPath", "command", "readyUrl", "identity", "environment", "launchArguments"]; // A web target grants no browser permissions yet, and a Windows app has none to grant. private static readonly string[] UnsupportedAppKeys = ["permissions"]; private static readonly string[] Environments = ["test", "staging", "production"]; private static readonly string[] Platforms = ["web", "ios", "android", "windows"]; private static readonly string[] AgentKeys = ["model", "judge", "provider", "system", "context", "baseUrl", "apiKeyEnv", "maxSteps", "maxModelCalls", "judgmentTimeout", "maxObservationBytes", "maxInputTokens", "providerOptions"]; private static readonly string[] UnsupportedAgentKeys = ["tools", "executor"]; private static readonly string[] CacheKeys = ["mode", "dir", "strict"]; private static readonly string[] UnsupportedCacheKeys = ["store"]; private E2EConfig() { } /// The file this config was read from, or null for the defaults. public string? ConfigPath { get; private init; } /// The config file's directory. Relative paths such as cache.dir resolve against it. public string ProjectRoot { get; private init; } = ""; /// True when the CI environment variable is set to a true value. public bool Ci { get; private init; } public IReadOnlyList Targets { get; private init; } = []; /// The target a test runs on. The port runs one target. public TargetConfig Target => Targets[0]; /// Whole-test timeout. Upstream timeout, 120 s by default. public TimeSpan Timeout { get; private init; } = E2EDefaults.TestTimeout; /// Engine start timeout. Upstream launchTimeout, 60 s by default. public TimeSpan LaunchTimeout { get; private init; } = E2EDefaults.LaunchTimeout; public TimeSpan ActionTimeout { get; private init; } = E2EDefaults.ActionTimeout; public TimeSpan AssertionTimeout { get; private init; } = E2EDefaults.AssertionTimeout; /// Engine shutdown timeout. Upstream cleanupTimeout, 30 s by default. public TimeSpan CleanupTimeout { get; private init; } = E2EDefaults.CleanupTimeout; /// 0 to 10. Defaults to 1 in CI and 0 elsewhere. NUnit retries still come from [Retry]. public int Retries { get; private init; } public IReadOnlyDictionary Agents { get; private init; } = ReadOnlyDictionary.Empty; /// The default agent. It always exists; its model may be null. public AgentConfig Agent => Agents["default"]; public CacheConfig Cache { get; private init; } = new(); public Secrets Secrets { get; private init; } = Secrets.Empty; /// Reads and validates a config file. Throws with INVALID_CONFIG. public static E2EConfig Load(string path, Func? environment = null) { ArgumentException.ThrowIfNullOrWhiteSpace(path); var full = Path.GetFullPath(path); string json; try { json = File.ReadAllText(full); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { throw new ConfigurationException("INVALID_CONFIG", "Cannot read " + full + ": " + ex.Message, ex); } return Parse(json, Path.GetDirectoryName(full)!, full, environment); } /// Validates config JSON. anchors relative paths. public static E2EConfig Parse(string json, string projectRoot, Func? environment = null) { ArgumentNullException.ThrowIfNull(json); ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot); return Parse(json, Path.GetFullPath(projectRoot), null, environment); } /// The config with no file: upstream defaults, a web target with no URL, and no model. public static E2EConfig Default(string projectRoot, Func? environment = null) { ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot); return Parse("{}", Path.GetFullPath(projectRoot), null, environment); } /// /// The nearest in each start directory or one of its parents, /// tried in order, or null. A directory with a .git entry is the repository root: the search /// looks for the file there and goes no higher. /// public static string? Find(params string[] startDirectories) { ArgumentNullException.ThrowIfNull(startDirectories); foreach (var start in startDirectories) { if (string.IsNullOrWhiteSpace(start)) { continue; } for (var directory = new DirectoryInfo(Path.GetFullPath(start)); directory is not null; directory = directory.Parent) { var candidate = Path.Combine(directory.FullName, FileName); if (File.Exists(candidate)) { return candidate; } if (Path.Exists(Path.Combine(directory.FullName, ".git"))) { break; } } } return null; } /// /// Loads the nearest above the test assembly directory, then /// above the working directory. Without one, returns for the working directory. /// public static E2EConfig Discover(Func? environment = null) { var path = Find(AppContext.BaseDirectory, Directory.GetCurrentDirectory()); return path is null ? Default(Directory.GetCurrentDirectory(), environment) : Load(path, environment); } /// True when CI is set to anything but empty, 0, or false. public static bool IsCi(Func? environment = null) { var value = (environment ?? Environment.GetEnvironmentVariable)("CI")?.Trim(); return !string.IsNullOrEmpty(value) && value != "0" && !string.Equals(value, "false", StringComparison.OrdinalIgnoreCase); } private static E2EConfig Parse(string json, string projectRoot, string? configPath, Func? environment) { environment ??= Environment.GetEnvironmentVariable; JsonDocument document; try { document = JsonDocument.Parse(json, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); } catch (JsonException ex) { throw Invalid((configPath ?? "config") + " is not valid JSON: " + ex.Message); } using (document) { var root = document.RootElement; if (root.ValueKind != JsonValueKind.Object) { throw Invalid("the config must be a JSON object"); } CheckKeys(root, "config", TopLevelKeys, UnsupportedTopLevelKeys, MovedKeys, RemovedTopLevelKeys); var ci = IsCi(environment); var trace = ReadRun(root, RecordingKind.Trace, environment); var video = ReadRun(root, RecordingKind.Video, environment); return new E2EConfig { ConfigPath = configPath, ProjectRoot = projectRoot, Ci = ci, Targets = ReadTargets(Property(root, "targets"), projectRoot, trace, video), Timeout = Milliseconds(root, "timeout", E2EDefaults.TestTimeout), LaunchTimeout = Milliseconds(root, "launchTimeout", E2EDefaults.LaunchTimeout), ActionTimeout = Milliseconds(root, "actionTimeout", E2EDefaults.ActionTimeout), AssertionTimeout = Milliseconds(root, "assertionTimeout", E2EDefaults.AssertionTimeout), CleanupTimeout = Milliseconds(root, "cleanupTimeout", E2EDefaults.CleanupTimeout), Retries = Property(root, "retries") is { } retries ? Integer(retries, "retries", 0, 10) : ci ? 1 : 0, Agents = ReadAgents(Property(root, "agents")), Cache = ReadCache(Property(root, "cache"), projectRoot, ci, environment), Secrets = ReadSecrets(Property(root, "secrets"), environment), }; } } private static List ReadTargets(JsonElement? raw, string projectRoot, RunRecording trace, RunRecording video) { if (raw is null) { return [new TargetConfig { Name = "web", Platform = "web", App = new AppConfig(), Trace = trace.For(null), Video = video.For(null) }]; } if (raw.Value.ValueKind != JsonValueKind.Array || raw.Value.GetArrayLength() == 0) { throw Invalid("targets must be a non-empty array"); } if (raw.Value.GetArrayLength() > 1) { throw Invalid("targets has " + raw.Value.GetArrayLength().ToString(CultureInfo.InvariantCulture) + " entries; the .NET port runs one target"); } var target = raw.Value[0]; if (target.ValueKind != JsonValueKind.Object) { throw Invalid("targets[0] must be an object"); } CheckKeys(target, "targets[0]", TargetKeys, UnsupportedTargetKeys, null); var platform = OptionalString(target, "platform", "targets[0].platform") ?? "web"; if (!Platforms.Contains(platform, StringComparer.Ordinal)) { throw Invalid("targets[0].platform \"" + platform + "\" is not supported; the .NET port drives \"web\", \"ios\", \"android\", and \"windows\""); } var name = OptionalString(target, "name", "targets[0].name") ?? platform; if (!ValidName(name)) { throw Invalid("targets[0].name must use ASCII letters, numbers, \"_\", \"-\", and \".\", and cannot be only dots"); } var ownTrace = ReadMode(Property(target, "trace"), "targets[0].trace", RecordingKind.Trace); var ownVideo = ReadMode(Property(target, "video"), "targets[0].video", RecordingKind.Video); var app = new AppConfig(); if (Property(target, "app") is { } rawApp) { if (rawApp.ValueKind != JsonValueKind.Object) { throw Invalid("targets[0].app must be an object"); } // A pinned-app target (a device or a Windows app) names its app by bundle id and build path, a web target by URL. var pinned = platform != "web"; CheckKeys(rawApp, "targets[0].app", !pinned ? AppKeys : platform == "windows" ? WindowsAppKeys : MobileAppKeys, UnsupportedAppKeys, null); var where = "target \"" + name + "\" app"; app = new AppConfig { Url = OptionalString(rawApp, "url", "targets[0].app.url") is { } url ? Urls.NormalizeBaseUrl(url).Href : null, BundleId = OptionalString(rawApp, "bundleId", "targets[0].app.bundleId"), AppPath = OptionalString(rawApp, "appPath", "targets[0].app.appPath"), Command = Property(rawApp, "command") is { } command ? Command.Normalize(command, "target \"" + name + "\" app.command") : null, ReadyUrl = OptionalString(rawApp, "readyUrl", "targets[0].app.readyUrl"), Identity = OptionalString(rawApp, "identity", "targets[0].app.identity"), Environment = ReadEnvironment(Property(rawApp, "environment"), where + ".environment"), LaunchArguments = ReadLaunchArguments(Property(rawApp, "launchArguments"), where + ".launchArguments"), Permissions = ReadPermissions(Property(rawApp, "permissions"), where + ".permissions"), }; // Resolved once here so a bad command, a free port nothing starts on, or a {port} without a URL fails at load, as upstream's resolveConfig does. TargetApps.ResolveTargetApp(name, app, projectRoot); } return [new TargetConfig { Name = name, Platform = platform, App = app, Trace = trace.For(ownTrace), Video = video.For(ownVideo) }]; } // The run's trace or video mode before any target speaks: the environment variable, the config root, and the default under both. // A failed attempt keeps its trace by default. The video is off. private static RunRecording ReadRun(JsonElement root, RecordingKind kind, Func environment) { var key = kind == RecordingKind.Trace ? "trace" : "video"; var variable = "E2E_" + key.ToUpperInvariant(); RecordingMode? flag = null; if (environment(variable) is { } text && !string.IsNullOrWhiteSpace(text)) { flag = ReadMode(JsonSerializer.SerializeToElement(text.Trim()), variable, kind); } return new RunRecording(flag, ReadMode(Property(root, key), key, kind), kind == RecordingKind.Trace ? RecordingMode.RetainOnFailure : RecordingMode.Off); } // Checks one trace or video value: a mode, or null when the key is unset. private static RecordingMode? ReadMode(JsonElement? raw, string where, RecordingKind kind) { if (raw is not { } value) { return null; } var text = value.ValueKind == JsonValueKind.String ? value.GetString() : null; if (RecordingModes.TryParse(text, out var mode)) { return mode; } if (kind == RecordingKind.Trace) { var block = value.ValueKind == JsonValueKind.Object; var record = block ? (value.TryGetProperty("record", out var inner) && inner.ValueKind == JsonValueKind.String ? inner.GetString() : null) : text; if (RecordingModes.TryLegacyTraceSpelling(record, block, out var was, out var meant)) { var spelled = where.StartsWith("E2E_", StringComparison.Ordinal) ? where + "=" + RecordingModes.Name(meant) : "trace: '" + RecordingModes.Name(meant) + "'"; throw Invalid(where + " " + was + " is the old spelling of " + spelled + "; the modes are " + RecordingModes.List); } } throw Invalid(where + " must be one of " + RecordingModes.List + ", got " + value.GetRawText()); } private static ReadOnlyDictionary ReadAgents(JsonElement? raw) { var agents = new Dictionary(StringComparer.Ordinal); if (raw is not null) { if (raw.Value.ValueKind != JsonValueKind.Object) { throw Invalid("agents must be an object of agents by name: agents: { \"default\": { \"model\": ... } }"); } foreach (var property in raw.Value.EnumerateObject()) { if (!ValidName(property.Name)) { throw Invalid("agent name \"" + property.Name + "\" must use ASCII letters, numbers, \"_\", \"-\", and \".\", and cannot be only dots"); } var label = "agents." + property.Name; var value = property.Value; if (value.ValueKind != JsonValueKind.Object) { throw Invalid(label + " must be an object"); } CheckKeys(value, label, AgentKeys, UnsupportedAgentKeys, null, RemovedAgentKeys); var context = AnyString(value, "context", label + ".context"); ResolvedAgent.CheckContext(context, label + ".context"); var provider = OptionalString(value, "provider", label + ".provider") ?? AgentConfig.DefaultProvider; if (!AgentConfig.Providers.Contains(provider, StringComparer.Ordinal)) { throw Invalid(label + ".provider must be one of " + string.Join(", ", AgentConfig.Providers.Select(name => "\"" + name + "\""))); } var baseUrl = OptionalString(value, "baseUrl", label + ".baseUrl"); var apiKeyEnv = OptionalString(value, "apiKeyEnv", label + ".apiKeyEnv"); if (AgentConfig.IsSubscription(provider) && (baseUrl is not null || apiKeyEnv is not null)) { throw Invalid(label + ": the " + provider + " subscription reads its stored login, so baseUrl and apiKeyEnv do not apply"); } if (provider == "openai-compatible" && baseUrl is null) { throw Invalid(label + ".baseUrl is required for provider \"openai-compatible\""); } agents[property.Name] = new AgentConfig { Model = OptionalString(value, "model", label + ".model"), Judge = OptionalString(value, "judge", label + ".judge"), Provider = provider, System = AnyString(value, "system", label + ".system"), Context = context, BaseUrl = baseUrl, ApiKeyEnv = apiKeyEnv, MaxSteps = Property(value, "maxSteps") is { } steps ? Integer(steps, label + ".maxSteps", 1, ResolvedAgent.MaxBudget) : null, MaxModelCalls = Property(value, "maxModelCalls") is { } calls ? Integer(calls, label + ".maxModelCalls", 1, ResolvedAgent.MaxBudget) : null, JudgmentTimeout = Property(value, "judgmentTimeout") is { } judgment ? TimeSpan.FromMilliseconds(Integer(judgment, label + ".judgmentTimeout", 1, int.MaxValue)) : E2EDefaults.JudgmentTimeout, MaxObservationBytes = Property(value, "maxObservationBytes") is { } bytes ? Integer(bytes, label + ".maxObservationBytes", ResolvedAgent.MinObservationBytes, ResolvedAgent.MaxObservationBytesLimit) : E2EDefaults.MaxObservationBytes, MaxInputTokens = Property(value, "maxInputTokens") is { } tokens ? Integer(tokens, label + ".maxInputTokens", 1, ResolvedAgent.MaxInputTokensLimit) : E2EDefaults.MaxInputTokens, ProviderOptions = ReadProviderOptions(Property(value, "providerOptions"), label), }; } } agents.TryAdd("default", new AgentConfig()); return new ReadOnlyDictionary(agents); } private static ReadOnlyDictionary? ReadProviderOptions(JsonElement? raw, string label) { if (raw is null) { return null; } if (raw.Value.ValueKind != JsonValueKind.Object) { throw Invalid(label + ".providerOptions must be an object of provider options by provider"); } var options = new Dictionary(StringComparer.Ordinal); foreach (var provider in raw.Value.EnumerateObject()) { if (provider.Value.ValueKind != JsonValueKind.Object) { throw Invalid(label + ".providerOptions." + provider.Name + " must be an object of provider options"); } options[provider.Name] = provider.Value.Clone(); } return new ReadOnlyDictionary(options); } private static CacheConfig ReadCache(JsonElement? raw, string projectRoot, bool ci, Func environment) { // A non-blank E2E_CACHE_DIR replaces cache.dir and the default; the mode stays as configured. var shared = environment("E2E_CACHE_DIR") is { } variable && !string.IsNullOrWhiteSpace(variable) ? variable : null; var sharedDirectory = shared is null ? null : Path.GetFullPath(Path.Combine(projectRoot, ExpandHome(shared))); var defaultDirectory = sharedDirectory ?? Path.Combine(projectRoot, ".e2e", "cache"); if (raw is null) { return new CacheConfig { Mode = ci ? CacheMode.ReadOnly : CacheMode.ReadWrite, Directory = defaultDirectory }; } if (raw.Value.ValueKind != JsonValueKind.Object) { throw Invalid("cache must be an object"); } var value = raw.Value; CheckKeys(value, "cache config", CacheKeys, UnsupportedCacheKeys, null); CacheMode mode; if (Property(value, "mode") is { } rawMode) { mode = (rawMode.ValueKind == JsonValueKind.String ? rawMode.GetString() : null) switch { "off" => CacheMode.Off, "read-only" => CacheMode.ReadOnly, "read-write" => CacheMode.ReadWrite, _ => throw Invalid("cache.mode must be \"off\", \"read-only\", or \"read-write\""), }; } else { // An unset mode is read-write locally. CI treats recordings as untrusted input and only reads them. mode = ci ? CacheMode.ReadOnly : CacheMode.ReadWrite; } var directory = defaultDirectory; if (Property(value, "dir") is { } rawDir) { var dir = rawDir.ValueKind == JsonValueKind.String ? rawDir.GetString() : null; if (string.IsNullOrWhiteSpace(dir)) { throw Invalid("cache.dir must be a non-empty path"); } directory = sharedDirectory ?? Path.GetFullPath(Path.Combine(projectRoot, dir)); } var strict = false; if (Property(value, "strict") is { } rawStrict) { strict = rawStrict.ValueKind switch { JsonValueKind.True => true, JsonValueKind.False => false, _ => throw Invalid("cache.strict must be a boolean"), }; } return new CacheConfig { Mode = mode, Directory = directory, Strict = strict }; } /// The value with a leading ~ or ~/ read as the home directory, which no shell expands in an env file. private static string ExpandHome(string value) { var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); if (value == "~") { return home; } return value.StartsWith("~/", StringComparison.Ordinal) || value.StartsWith("~" + Path.DirectorySeparatorChar, StringComparison.Ordinal) ? Path.Join(home, value[2..]) : value; } private static Secrets ReadSecrets(JsonElement? raw, Func environment) { if (raw is null) { return Secrets.Empty; } if (raw.Value.ValueKind != JsonValueKind.Object) { throw Invalid("secrets must be an object of secret values by name"); } var values = new Dictionary(StringComparer.Ordinal); foreach (var property in raw.Value.EnumerateObject()) { if (property.Name.Trim().Length == 0) { throw Invalid("secret names must be non-empty"); } } // One E2E_SECRET_* value would replace every secret that maps to the variable, set or not. var shared = raw.Value.EnumerateObject() .GroupBy(property => Secrets.EnvironmentVariable(property.Name), StringComparer.Ordinal) .FirstOrDefault(group => group.Count() > 1); if (shared is not null) { var names = shared.Select(property => "\"" + property.Name + "\"").ToArray(); throw Invalid( "secrets " + string.Join(", ", names[..^1]) + " and " + names[^1] + " share the override variable " + shared.Key + ", so one value set there would replace all of them; rename all but one"); } foreach (var property in raw.Value.EnumerateObject()) { var name = property.Name; var variable = Secrets.EnvironmentVariable(name); string? entry = property.Value.ValueKind switch { JsonValueKind.String => property.Value.GetString(), JsonValueKind.Null => null, _ => throw Invalid("secret \"" + name + "\" must be a non-empty string, or null to read only " + variable), }; var value = environment(variable) ?? entry; if (value is null) { throw Invalid("secret \"" + name + "\" has no value; set " + variable); } if (Secret.Problem(value) is { } problem) { throw Invalid("secret \"" + name + "\" " + problem); } values[name] = Secret.Create(name, value); } return new Secrets(values); } private static void CheckKeys(JsonElement value, string label, string[] allowed, string[] unsupported, Dictionary? moved, Dictionary? removed = null) { foreach (var property in value.EnumerateObject()) { var key = property.Name; if (removed is not null && removed.TryGetValue(key, out var replacement)) { throw Invalid((label == "config" ? key : label + "." + key) + " was removed: " + replacement); } if (allowed.Contains(key, StringComparer.Ordinal)) { continue; } var where = label == "config" ? "config key" : label + " key"; if (unsupported.Contains(key, StringComparer.Ordinal)) { throw Invalid(where + " \"" + key + "\" is not supported by the .NET port" + (key == "executor" ? ": an executor is code, so set AgentOptions.Executor" : "")); } string hint; if (moved is not null && moved.TryGetValue(key, out var movedHint)) { hint = "; " + movedHint; } else { hint = Suggestions.DidYouMean(key, allowed); } throw Invalid("unknown " + where + " \"" + key + "\"" + hint); } } private static JsonElement? Property(JsonElement value, string name) { return value.TryGetProperty(name, out var property) ? property : null; } private static string? OptionalString(JsonElement value, string name, string label) { if (Property(value, name) is not { } property) { return null; } var text = property.ValueKind == JsonValueKind.String ? property.GetString() : null; if (string.IsNullOrWhiteSpace(text)) { throw Invalid(label + " must be a non-empty string"); } return text; } // Checks app.environment: one of test, staging, production. private static string? ReadEnvironment(JsonElement? raw, string label) { if (raw is not { } value) { return null; } var text = value.ValueKind == JsonValueKind.String ? value.GetString() : null; return text is not null && Environments.Contains(text, StringComparer.Ordinal) ? text : throw Invalid(label + " must be one of " + string.Join(", ", Environments) + ", got " + value.GetRawText()); } // Checks app.launchArguments: an array of strings. private static ReadOnlyCollection? ReadLaunchArguments(JsonElement? raw, string label) { if (raw is not { } value) { return null; } if (value.ValueKind != JsonValueKind.Array || value.EnumerateArray().Any(argument => argument.ValueKind != JsonValueKind.String)) { throw Invalid(label + " must be an array of strings"); } return value.EnumerateArray().Select(argument => argument.GetString()!).ToList().AsReadOnly(); } // Checks app.permissions: an object of permission name to grant, deny, or reset. private static ReadOnlyDictionary? ReadPermissions(JsonElement? raw, string label) { if (raw is not { } value) { return null; } if (value.ValueKind != JsonValueKind.Object) { throw Invalid(label + " must be an object of permission name to grant, deny, or reset"); } var permissions = new Dictionary(StringComparer.Ordinal); foreach (var property in value.EnumerateObject()) { var state = property.Value.ValueKind == JsonValueKind.String ? property.Value.GetString() : null; permissions[property.Name] = state switch { "grant" => AppPermissionState.Grant, "deny" => AppPermissionState.Deny, "reset" => AppPermissionState.Reset, _ => throw Invalid(label + "." + property.Name + " must be grant, deny, or reset, got " + property.Value.GetRawText()), }; } return new ReadOnlyDictionary(permissions); } // A string that may be empty, as upstream system and context. private static string? AnyString(JsonElement value, string name, string label) { if (Property(value, name) is not { } property) { return null; } if (property.ValueKind != JsonValueKind.String) { throw Invalid(label + " must be a string"); } return property.GetString(); } private static TimeSpan Milliseconds(JsonElement root, string name, TimeSpan fallback) { return Property(root, name) is { } raw ? TimeSpan.FromMilliseconds(Integer(raw, name, 1, int.MaxValue)) : fallback; } private static int Integer(JsonElement value, string label, int min, int max) { if (value.ValueKind != JsonValueKind.Number || !value.TryGetInt32(out var number) || number < min || number > max) { throw Invalid(max == int.MaxValue ? label + " must be a positive integer" : label + " must be an integer from " + min.ToString(CultureInfo.InvariantCulture) + " through " + max.ToString(CultureInfo.InvariantCulture)); } return number; } private static bool ValidName(string name) { return name.Length > 0 && name.Any(ch => ch != '.') && name.All(ch => char.IsAsciiLetterOrDigit(ch) || ch is '_' or '-' or '.'); } private static ConfigurationException Invalid(string message) => new("INVALID_CONFIG", message); } /// One entry of targets. The port drives the web, ios, and android platforms. public sealed class TargetConfig { public required string Name { get; init; } public required string Platform { get; init; } public required AppConfig App { get; init; } /// This target with another app: the one resolved on the port its command was started on. internal TargetConfig WithApp(AppConfig app) => new() { Name = Name, Platform = Platform, App = app, Trace = Trace, Video = Video }; /// Which attempts write trace.md: E2E_TRACE, else this target's trace, else the config's, else retain-on-failure. public ResolvedRecording Trace { get; init; } = new(RecordingMode.RetainOnFailure, RecordingSource.Default); /// Which attempts record a video: E2E_VIDEO, else this target's video, else the config's, else off. public ResolvedRecording Video { get; init; } = new(RecordingMode.Off, RecordingSource.Default); } /// One kind's modes before any target speaks: the environment variable, the config root, and the default under both. internal sealed record RunRecording(RecordingMode? Flag, RecordingMode? Config, RecordingMode Fallback) { /// A target's effective mode and where it came from: the variable, else the target's own, else the config root's, else the default. public ResolvedRecording For(RecordingMode? own) { if (Flag is { } flag) { return new ResolvedRecording(flag, RecordingSource.Run); } if (own is { } target) { return new ResolvedRecording(target, RecordingSource.Target); } return Config is { } config ? new ResolvedRecording(config, RecordingSource.Run) : new ResolvedRecording(Fallback, RecordingSource.Default); } } public sealed class AppConfig { /// /// Base URL for App.OpenAsync, normalized: a missing scheme becomes https://, or http:// for a loopback host. /// Relative URLs resolve against it. /// public string? Url { get; init; } /// The bundle id (iOS) or package (Android) of an installed app a device target launches. public string? BundleId { get; init; } /// The build a device target installs, relative to the folder of e2e.config.json. public string? AppPath { get; init; } /// /// The command that starts the app (upstream's app.command), started before the first test and stopped when the process ends. /// {port} in and is the port the app is served on. A /// with port 0 asks for a free one, and needs this. /// public CommandConfig? Command { get; init; } /// The URL that answers once has started the app (upstream's app.readyUrl). Unset, . public string? ReadyUrl { get; init; } /// /// The stable identity that keys the replay cache and saved sessions (upstream's app.identity). Unset, the URL's origin and /// path as declared, else , else . An ephemeral per-deploy origin (a PR preview) can /// declare the identity of the app it is a deployment of, and so share its entries. /// public string? Identity { get; init; } /// /// What the app is: test, staging, or production (upstream's app.environment). Unset, test for a /// loopback, .localhost, or .test host, and production for any other host. The environment always joins the /// cache key, so an identity never bleeds entries across environments. /// public string? Environment { get; init; } /// Arguments every fresh launch of the app takes: process arguments on iOS, am start arguments on Android, the command line on Windows. public IReadOnlyList? LaunchArguments { get; init; } /// /// Permissions the app holds before every fresh launch, by permission name (upstream's app.permissions). A device target /// takes the names of DevicePermission. /// public IReadOnlyDictionary? Permissions { get; init; } /// This app with another URL: the one on the port its command was started on. internal AppConfig WithUrl(string? url) => new() { Url = url, BundleId = BundleId, AppPath = AppPath, Identity = Identity, Environment = Environment, LaunchArguments = LaunchArguments, Permissions = Permissions, Command = Command, ReadyUrl = ReadyUrl, }; } /// What a permission is set to before a launch (upstream's AppPermissionState): held, refused, or not asked for yet, so the OS asks again. public enum AppPermissionState { /// The app holds the permission. Grant, /// The permission is refused. Deny, /// The permission is not asked for yet. Reset, } /// /// One entry of agents. Upstream model and judge are AI SDK instances; here they are model ids /// for , which share and . /// Each entry starts from the defaults; none inherits another's values. /// public sealed class AgentConfig { public const string DefaultProvider = "openai"; /// /// The provider values: API keys (openai, openai-responses, azure, anthropic, /// google, bedrock, xai, openrouter, gateway, openai-compatible) and /// subscription logins (chatgpt, copilot, grok, opencode-console). /// public static IReadOnlyList Providers { get; } = [ "openai", "openai-responses", "azure", "anthropic", "google", "bedrock", "xai", "openrouter", "gateway", "openai-compatible", "chatgpt", "copilot", "grok", "opencode-console", ]; public string? Model { get; init; } /// The model id that judges assert, waitFor, and extract. Unset, the judge is . public string? Judge { get; init; } /// Who serves and . One of ; openai by default. public string Provider { get; init; } = DefaultProvider; /// Text appended to the act rules. Judges never see it. public string? System { get; init; } /// Project context told to every model call, at most 16384 UTF-8 bytes. public string? Context { get; init; } /// The API root. Unset, the provider's own; for azure it is built from AZURE_RESOURCE_NAME. public string? BaseUrl { get; init; } /// The environment variable holding the key. Unset, the one the provider reads, such as ANTHROPIC_API_KEY. public string? ApiKeyEnv { get; init; } /// 1 through 100. Null when the entry leaves it out: applies, and Explore sets its own. public int? MaxSteps { get; init; } /// 1 through 100. Null when the entry leaves it out: applies, and Explore sets its own. public int? MaxModelCalls { get; init; } public TimeSpan JudgmentTimeout { get; init; } = E2EDefaults.JudgmentTimeout; /// 1024 through 16777216. See . public int MaxObservationBytes { get; init; } = E2EDefaults.MaxObservationBytes; /// 1 through 1000000. See . public int MaxInputTokens { get; init; } = E2EDefaults.MaxInputTokens; /// Provider options by provider. Each client adds the entry under its own key to its request body. public IReadOnlyDictionary? ProviderOptions { get; init; } /// The model for , or null when no model is set. public IAgentModel? CreateModel() => Create(Model); /// The model for , or null when no judge is set. public IAgentModel? CreateJudge() => Create(Judge); /// The this entry describes, with its models created. public AgentOptions CreateOptions() { return new AgentOptions { Model = CreateModel(), Judge = CreateJudge(), System = System, Context = Context, MaxSteps = MaxSteps, MaxModelCalls = MaxModelCalls, JudgmentTimeout = JudgmentTimeout, MaxObservationBytes = MaxObservationBytes, MaxInputTokens = MaxInputTokens, ProviderOptions = ProviderOptions, }; } internal static bool IsSubscription(string provider) => provider is "chatgpt" or "copilot" or "grok" or "opencode-console"; private IAgentModel? Create(string? model) { if (model is null) { return null; } return Provider switch { "openai-responses" => new OpenAiResponsesModel(new OpenAiResponsesModelOptions { Model = model, BaseUrl = BaseUrl ?? "https://api.openai.com/v1", ApiKeyEnv = ApiKeyEnv ?? "OPENAI_API_KEY", }), "azure" => new OpenAiResponsesModel(new OpenAiResponsesModelOptions { Model = model, BaseUrl = BaseUrl ?? AzureBaseUrl(), Provider = "azure", ApiKeyEnv = ApiKeyEnv ?? "AZURE_API_KEY", ApiKeyHeader = "api-key", QueryParameters = new Dictionary(StringComparer.Ordinal) { ["api-version"] = "v1" }, }), "anthropic" => new AnthropicModel(new AnthropicModelOptions { Model = model, BaseUrl = BaseUrl ?? "https://api.anthropic.com/v1", ApiKeyEnv = ApiKeyEnv ?? "ANTHROPIC_API_KEY", }), "google" => new GoogleModel(new GoogleModelOptions { Model = model, BaseUrl = BaseUrl ?? "https://generativelanguage.googleapis.com/v1beta", ApiKeyEnv = ApiKeyEnv ?? "GOOGLE_GENERATIVE_AI_API_KEY", }), "bedrock" => new BedrockModel(new BedrockModelOptions { Model = model, BaseUrl = BaseUrl, ApiKeyEnv = ApiKeyEnv ?? "AWS_BEARER_TOKEN_BEDROCK", }), "xai" => Compatible(model, "https://api.x.ai/v1", "xai", "XAI_API_KEY", null), "openrouter" => Compatible(model, "https://openrouter.ai/api/v1", "openrouter", "OPENROUTER_API_KEY", null), "gateway" => Compatible(model, "https://ai-gateway.vercel.sh/v1", "gateway", "AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"), "openai-compatible" => Compatible(model, "http://127.0.0.1:11434/v1", "openai-compatible", "LLM_API_KEY", null), "chatgpt" => OAuth.Subscriptions.ChatGpt(model), "copilot" => OAuth.Subscriptions.Copilot(model), "grok" => OAuth.Subscriptions.Grok(model), "opencode-console" => OAuth.Subscriptions.OpenCodeConsole(model), _ => Compatible(model, "https://api.openai.com/v1", "openai", "OPENAI_API_KEY", null), }; } private OpenAiCompatibleModel Compatible(string model, string baseUrl, string provider, string apiKeyEnv, string? fallbackEnv) { return new OpenAiCompatibleModel(new OpenAiCompatibleModelOptions { Model = model, BaseUrl = BaseUrl ?? baseUrl, Provider = provider, ApiKeyEnv = ApiKeyEnv ?? apiKeyEnv, ApiKeyFallbackEnv = ApiKeyEnv is null ? fallbackEnv : null, }); } private static string AzureBaseUrl() { var resource = Environment.GetEnvironmentVariable("AZURE_RESOURCE_NAME"); return string.IsNullOrWhiteSpace(resource) ? throw new ConfigurationException("INVALID_CONFIG", "provider \"azure\" needs baseUrl or AZURE_RESOURCE_NAME") : "https://" + resource.Trim() + ".openai.azure.com/openai/v1"; } } public sealed class CacheConfig { /// Unset, it is , or in CI. public CacheMode Mode { get; init; } = CacheMode.ReadWrite; /// Absolute cache directory. cache.dir resolves against the config directory; the default is .e2e/cache. A non-blank E2E_CACHE_DIR replaces both. public string Directory { get; init; } = Path.Combine(".e2e", "cache"); /// When true, a recording that no longer matches fails with REPLAY_STALE instead of running live. public bool Strict { get; init; } } /// How the replay cache is used. public enum CacheMode { /// Never read or write recordings. Off, /// Replay recordings, never write or delete them. ReadOnly, /// Replay recordings, record verified acts, and delete unverified ones. ReadWrite, }