// Copyright 2026 TesterArmy.
// Modified by Dario Kondratiuk.
// SPDX-License-Identifier: Apache-2.0
using System.Collections.ObjectModel;
using System.Globalization;
using System.Text.Json;
using E2E.Internal;
namespace E2E;
///
/// The resolved e2e.config.json. The JSON shape follows upstream e2e.config.ts:
/// targets[].app.url, named agents, top-level timeouts in milliseconds,
/// retries, cache, and secrets. Unknown keys fail with
/// INVALID_CONFIG. E2ETest finds and applies this file; a fixture can
/// still override each value. There is no default model.
///
public sealed class E2EConfig
{
/// The file name looks for.
public const string FileName = "e2e.config.json";
private static readonly Dictionary MovedKeys = new(StringComparer.Ordinal)
{
["app"] = "the app under test is declared on its target: targets: [{ \"app\": { \"url\": ... } }]",
["engine"] = "the engine is chosen by the fixture; declare the target as targets: [{ \"platform\": \"web\" }]",
["agent"] = "agents are named: agents: { \"default\": { \"model\": ... } }",
["timeouts"] = "timeouts are top-level milliseconds: timeout, launchTimeout, actionTimeout, assertionTimeout, cleanupTimeout",
};
// Upstream's REMOVED_TOP_LEVEL_KEYS: keys the runner used to accept, each mapped to what replaces it.
private static readonly Dictionary RemovedTopLevelKeys = new(StringComparer.Ordinal)
{
["specVersion"] = "delete it; the runner version is the format version",
["limits"] = "set maxInputTokens on each agent (it was limits.maxModelTokensPerCall); the runner fixes maxAgentContextBytes, maxLedgerBytes, and maxEventsPerStep",
};
// Upstream's REMOVED_AGENT_KEYS: agent keys the runner used to accept, each mapped to what replaces it.
private static readonly Dictionary RemovedAgentKeys = new(StringComparer.Ordinal)
{
["timeout"] = "use judgmentTimeout, the deadline of one assert, waitFor, or extract call",
["maxTurns"] = "use maxModelCalls, the model requests one agent call may make",
["maxModelTokensPerCall"] = "use maxInputTokens",
["limits"] = "set maxInputTokens on the agent; the runner fixes the other limits",
};
private static readonly string[] TopLevelKeys =
[
"targets", "timeout", "launchTimeout", "actionTimeout", "assertionTimeout", "cleanupTimeout",
"retries", "agents", "cache", "secrets", "trace", "video",
];
// Upstream keys the .NET port does not implement. They fail loudly rather than being ignored.
private static readonly string[] UnsupportedTopLevelKeys =
[
"projectId", "tests", "failOnSkippedFailure", "workers", "artifacts", "output", "reporters", "credentials",
];
private static readonly string[] TargetKeys = ["name", "platform", "app", "trace", "video"];
private static readonly string[] UnsupportedTargetKeys = ["engine"];
private static readonly string[] AppKeys = ["url", "command", "readyUrl", "identity", "environment"];
private static readonly string[] MobileAppKeys =
["bundleId", "appPath", "command", "readyUrl", "identity", "environment", "launchArguments", "permissions"];
private static readonly string[] WindowsAppKeys =
["bundleId", "appPath", "command", "readyUrl", "identity", "environment", "launchArguments"];
// A web target grants no browser permissions yet, and a Windows app has none to grant.
private static readonly string[] UnsupportedAppKeys = ["permissions"];
private static readonly string[] Environments = ["test", "staging", "production"];
private static readonly string[] Platforms = ["web", "ios", "android", "windows"];
private static readonly string[] AgentKeys =
["model", "judge", "provider", "system", "context", "baseUrl", "apiKeyEnv", "maxSteps", "maxModelCalls", "judgmentTimeout", "maxObservationBytes", "maxInputTokens", "providerOptions"];
private static readonly string[] UnsupportedAgentKeys = ["tools", "executor"];
private static readonly string[] CacheKeys = ["mode", "dir", "strict"];
private static readonly string[] UnsupportedCacheKeys = ["store"];
private E2EConfig()
{
}
/// The file this config was read from, or null for the defaults.
public string? ConfigPath { get; private init; }
/// The config file's directory. Relative paths such as cache.dir resolve against it.
public string ProjectRoot { get; private init; } = "";
/// True when the CI environment variable is set to a true value.
public bool Ci { get; private init; }
public IReadOnlyList Targets { get; private init; } = [];
/// The target a test runs on. The port runs one target.
public TargetConfig Target => Targets[0];
/// Whole-test timeout. Upstream timeout, 120 s by default.
public TimeSpan Timeout { get; private init; } = E2EDefaults.TestTimeout;
/// Engine start timeout. Upstream launchTimeout, 60 s by default.
public TimeSpan LaunchTimeout { get; private init; } = E2EDefaults.LaunchTimeout;
public TimeSpan ActionTimeout { get; private init; } = E2EDefaults.ActionTimeout;
public TimeSpan AssertionTimeout { get; private init; } = E2EDefaults.AssertionTimeout;
/// Engine shutdown timeout. Upstream cleanupTimeout, 30 s by default.
public TimeSpan CleanupTimeout { get; private init; } = E2EDefaults.CleanupTimeout;
/// 0 to 10. Defaults to 1 in CI and 0 elsewhere. NUnit retries still come from [Retry].
public int Retries { get; private init; }
public IReadOnlyDictionary Agents { get; private init; } = ReadOnlyDictionary.Empty;
/// The default agent. It always exists; its model may be null.
public AgentConfig Agent => Agents["default"];
public CacheConfig Cache { get; private init; } = new();
public Secrets Secrets { get; private init; } = Secrets.Empty;
/// Reads and validates a config file. Throws with INVALID_CONFIG.
public static E2EConfig Load(string path, Func? environment = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(path);
var full = Path.GetFullPath(path);
string json;
try
{
json = File.ReadAllText(full);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
throw new ConfigurationException("INVALID_CONFIG", "Cannot read " + full + ": " + ex.Message, ex);
}
return Parse(json, Path.GetDirectoryName(full)!, full, environment);
}
/// Validates config JSON. anchors relative paths.
public static E2EConfig Parse(string json, string projectRoot, Func? environment = null)
{
ArgumentNullException.ThrowIfNull(json);
ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot);
return Parse(json, Path.GetFullPath(projectRoot), null, environment);
}
/// The config with no file: upstream defaults, a web target with no URL, and no model.
public static E2EConfig Default(string projectRoot, Func? environment = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot);
return Parse("{}", Path.GetFullPath(projectRoot), null, environment);
}
///
/// The nearest in each start directory or one of its parents,
/// tried in order, or null. A directory with a .git entry is the repository root: the search
/// looks for the file there and goes no higher.
///
public static string? Find(params string[] startDirectories)
{
ArgumentNullException.ThrowIfNull(startDirectories);
foreach (var start in startDirectories)
{
if (string.IsNullOrWhiteSpace(start))
{
continue;
}
for (var directory = new DirectoryInfo(Path.GetFullPath(start)); directory is not null; directory = directory.Parent)
{
var candidate = Path.Combine(directory.FullName, FileName);
if (File.Exists(candidate))
{
return candidate;
}
if (Path.Exists(Path.Combine(directory.FullName, ".git")))
{
break;
}
}
}
return null;
}
///
/// Loads the nearest above the test assembly directory, then
/// above the working directory. Without one, returns for the working directory.
///
public static E2EConfig Discover(Func? environment = null)
{
var path = Find(AppContext.BaseDirectory, Directory.GetCurrentDirectory());
return path is null ? Default(Directory.GetCurrentDirectory(), environment) : Load(path, environment);
}
/// True when CI is set to anything but empty, 0, or false.
public static bool IsCi(Func? environment = null)
{
var value = (environment ?? Environment.GetEnvironmentVariable)("CI")?.Trim();
return !string.IsNullOrEmpty(value)
&& value != "0"
&& !string.Equals(value, "false", StringComparison.OrdinalIgnoreCase);
}
private static E2EConfig Parse(string json, string projectRoot, string? configPath, Func? environment)
{
environment ??= Environment.GetEnvironmentVariable;
JsonDocument document;
try
{
document = JsonDocument.Parse(json, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
}
catch (JsonException ex)
{
throw Invalid((configPath ?? "config") + " is not valid JSON: " + ex.Message);
}
using (document)
{
var root = document.RootElement;
if (root.ValueKind != JsonValueKind.Object)
{
throw Invalid("the config must be a JSON object");
}
CheckKeys(root, "config", TopLevelKeys, UnsupportedTopLevelKeys, MovedKeys, RemovedTopLevelKeys);
var ci = IsCi(environment);
var trace = ReadRun(root, RecordingKind.Trace, environment);
var video = ReadRun(root, RecordingKind.Video, environment);
return new E2EConfig
{
ConfigPath = configPath,
ProjectRoot = projectRoot,
Ci = ci,
Targets = ReadTargets(Property(root, "targets"), projectRoot, trace, video),
Timeout = Milliseconds(root, "timeout", E2EDefaults.TestTimeout),
LaunchTimeout = Milliseconds(root, "launchTimeout", E2EDefaults.LaunchTimeout),
ActionTimeout = Milliseconds(root, "actionTimeout", E2EDefaults.ActionTimeout),
AssertionTimeout = Milliseconds(root, "assertionTimeout", E2EDefaults.AssertionTimeout),
CleanupTimeout = Milliseconds(root, "cleanupTimeout", E2EDefaults.CleanupTimeout),
Retries = Property(root, "retries") is { } retries ? Integer(retries, "retries", 0, 10) : ci ? 1 : 0,
Agents = ReadAgents(Property(root, "agents")),
Cache = ReadCache(Property(root, "cache"), projectRoot, ci, environment),
Secrets = ReadSecrets(Property(root, "secrets"), environment),
};
}
}
private static List ReadTargets(JsonElement? raw, string projectRoot, RunRecording trace, RunRecording video)
{
if (raw is null)
{
return [new TargetConfig { Name = "web", Platform = "web", App = new AppConfig(), Trace = trace.For(null), Video = video.For(null) }];
}
if (raw.Value.ValueKind != JsonValueKind.Array || raw.Value.GetArrayLength() == 0)
{
throw Invalid("targets must be a non-empty array");
}
if (raw.Value.GetArrayLength() > 1)
{
throw Invalid("targets has " + raw.Value.GetArrayLength().ToString(CultureInfo.InvariantCulture) + " entries; the .NET port runs one target");
}
var target = raw.Value[0];
if (target.ValueKind != JsonValueKind.Object)
{
throw Invalid("targets[0] must be an object");
}
CheckKeys(target, "targets[0]", TargetKeys, UnsupportedTargetKeys, null);
var platform = OptionalString(target, "platform", "targets[0].platform") ?? "web";
if (!Platforms.Contains(platform, StringComparer.Ordinal))
{
throw Invalid("targets[0].platform \"" + platform + "\" is not supported; the .NET port drives \"web\", \"ios\", \"android\", and \"windows\"");
}
var name = OptionalString(target, "name", "targets[0].name") ?? platform;
if (!ValidName(name))
{
throw Invalid("targets[0].name must use ASCII letters, numbers, \"_\", \"-\", and \".\", and cannot be only dots");
}
var ownTrace = ReadMode(Property(target, "trace"), "targets[0].trace", RecordingKind.Trace);
var ownVideo = ReadMode(Property(target, "video"), "targets[0].video", RecordingKind.Video);
var app = new AppConfig();
if (Property(target, "app") is { } rawApp)
{
if (rawApp.ValueKind != JsonValueKind.Object)
{
throw Invalid("targets[0].app must be an object");
}
// A pinned-app target (a device or a Windows app) names its app by bundle id and build path, a web target by URL.
var pinned = platform != "web";
CheckKeys(rawApp, "targets[0].app", !pinned ? AppKeys : platform == "windows" ? WindowsAppKeys : MobileAppKeys, UnsupportedAppKeys, null);
var where = "target \"" + name + "\" app";
app = new AppConfig
{
Url = OptionalString(rawApp, "url", "targets[0].app.url") is { } url ? Urls.NormalizeBaseUrl(url).Href : null,
BundleId = OptionalString(rawApp, "bundleId", "targets[0].app.bundleId"),
AppPath = OptionalString(rawApp, "appPath", "targets[0].app.appPath"),
Command = Property(rawApp, "command") is { } command ? Command.Normalize(command, "target \"" + name + "\" app.command") : null,
ReadyUrl = OptionalString(rawApp, "readyUrl", "targets[0].app.readyUrl"),
Identity = OptionalString(rawApp, "identity", "targets[0].app.identity"),
Environment = ReadEnvironment(Property(rawApp, "environment"), where + ".environment"),
LaunchArguments = ReadLaunchArguments(Property(rawApp, "launchArguments"), where + ".launchArguments"),
Permissions = ReadPermissions(Property(rawApp, "permissions"), where + ".permissions"),
};
// Resolved once here so a bad command, a free port nothing starts on, or a {port} without a URL fails at load, as upstream's resolveConfig does.
TargetApps.ResolveTargetApp(name, app, projectRoot);
}
return [new TargetConfig { Name = name, Platform = platform, App = app, Trace = trace.For(ownTrace), Video = video.For(ownVideo) }];
}
// The run's trace or video mode before any target speaks: the environment variable, the config root, and the default under both.
// A failed attempt keeps its trace by default. The video is off.
private static RunRecording ReadRun(JsonElement root, RecordingKind kind, Func environment)
{
var key = kind == RecordingKind.Trace ? "trace" : "video";
var variable = "E2E_" + key.ToUpperInvariant();
RecordingMode? flag = null;
if (environment(variable) is { } text && !string.IsNullOrWhiteSpace(text))
{
flag = ReadMode(JsonSerializer.SerializeToElement(text.Trim()), variable, kind);
}
return new RunRecording(flag, ReadMode(Property(root, key), key, kind), kind == RecordingKind.Trace ? RecordingMode.RetainOnFailure : RecordingMode.Off);
}
// Checks one trace or video value: a mode, or null when the key is unset.
private static RecordingMode? ReadMode(JsonElement? raw, string where, RecordingKind kind)
{
if (raw is not { } value)
{
return null;
}
var text = value.ValueKind == JsonValueKind.String ? value.GetString() : null;
if (RecordingModes.TryParse(text, out var mode))
{
return mode;
}
if (kind == RecordingKind.Trace)
{
var block = value.ValueKind == JsonValueKind.Object;
var record = block ? (value.TryGetProperty("record", out var inner) && inner.ValueKind == JsonValueKind.String ? inner.GetString() : null) : text;
if (RecordingModes.TryLegacyTraceSpelling(record, block, out var was, out var meant))
{
var spelled = where.StartsWith("E2E_", StringComparison.Ordinal) ? where + "=" + RecordingModes.Name(meant) : "trace: '" + RecordingModes.Name(meant) + "'";
throw Invalid(where + " " + was + " is the old spelling of " + spelled + "; the modes are " + RecordingModes.List);
}
}
throw Invalid(where + " must be one of " + RecordingModes.List + ", got " + value.GetRawText());
}
private static ReadOnlyDictionary ReadAgents(JsonElement? raw)
{
var agents = new Dictionary(StringComparer.Ordinal);
if (raw is not null)
{
if (raw.Value.ValueKind != JsonValueKind.Object)
{
throw Invalid("agents must be an object of agents by name: agents: { \"default\": { \"model\": ... } }");
}
foreach (var property in raw.Value.EnumerateObject())
{
if (!ValidName(property.Name))
{
throw Invalid("agent name \"" + property.Name + "\" must use ASCII letters, numbers, \"_\", \"-\", and \".\", and cannot be only dots");
}
var label = "agents." + property.Name;
var value = property.Value;
if (value.ValueKind != JsonValueKind.Object)
{
throw Invalid(label + " must be an object");
}
CheckKeys(value, label, AgentKeys, UnsupportedAgentKeys, null, RemovedAgentKeys);
var context = AnyString(value, "context", label + ".context");
ResolvedAgent.CheckContext(context, label + ".context");
var provider = OptionalString(value, "provider", label + ".provider") ?? AgentConfig.DefaultProvider;
if (!AgentConfig.Providers.Contains(provider, StringComparer.Ordinal))
{
throw Invalid(label + ".provider must be one of " + string.Join(", ", AgentConfig.Providers.Select(name => "\"" + name + "\"")));
}
var baseUrl = OptionalString(value, "baseUrl", label + ".baseUrl");
var apiKeyEnv = OptionalString(value, "apiKeyEnv", label + ".apiKeyEnv");
if (AgentConfig.IsSubscription(provider) && (baseUrl is not null || apiKeyEnv is not null))
{
throw Invalid(label + ": the " + provider + " subscription reads its stored login, so baseUrl and apiKeyEnv do not apply");
}
if (provider == "openai-compatible" && baseUrl is null)
{
throw Invalid(label + ".baseUrl is required for provider \"openai-compatible\"");
}
agents[property.Name] = new AgentConfig
{
Model = OptionalString(value, "model", label + ".model"),
Judge = OptionalString(value, "judge", label + ".judge"),
Provider = provider,
System = AnyString(value, "system", label + ".system"),
Context = context,
BaseUrl = baseUrl,
ApiKeyEnv = apiKeyEnv,
MaxSteps = Property(value, "maxSteps") is { } steps ? Integer(steps, label + ".maxSteps", 1, ResolvedAgent.MaxBudget) : null,
MaxModelCalls = Property(value, "maxModelCalls") is { } calls ? Integer(calls, label + ".maxModelCalls", 1, ResolvedAgent.MaxBudget) : null,
JudgmentTimeout = Property(value, "judgmentTimeout") is { } judgment
? TimeSpan.FromMilliseconds(Integer(judgment, label + ".judgmentTimeout", 1, int.MaxValue))
: E2EDefaults.JudgmentTimeout,
MaxObservationBytes = Property(value, "maxObservationBytes") is { } bytes
? Integer(bytes, label + ".maxObservationBytes", ResolvedAgent.MinObservationBytes, ResolvedAgent.MaxObservationBytesLimit)
: E2EDefaults.MaxObservationBytes,
MaxInputTokens = Property(value, "maxInputTokens") is { } tokens
? Integer(tokens, label + ".maxInputTokens", 1, ResolvedAgent.MaxInputTokensLimit)
: E2EDefaults.MaxInputTokens,
ProviderOptions = ReadProviderOptions(Property(value, "providerOptions"), label),
};
}
}
agents.TryAdd("default", new AgentConfig());
return new ReadOnlyDictionary(agents);
}
private static ReadOnlyDictionary? ReadProviderOptions(JsonElement? raw, string label)
{
if (raw is null)
{
return null;
}
if (raw.Value.ValueKind != JsonValueKind.Object)
{
throw Invalid(label + ".providerOptions must be an object of provider options by provider");
}
var options = new Dictionary(StringComparer.Ordinal);
foreach (var provider in raw.Value.EnumerateObject())
{
if (provider.Value.ValueKind != JsonValueKind.Object)
{
throw Invalid(label + ".providerOptions." + provider.Name + " must be an object of provider options");
}
options[provider.Name] = provider.Value.Clone();
}
return new ReadOnlyDictionary(options);
}
private static CacheConfig ReadCache(JsonElement? raw, string projectRoot, bool ci, Func environment)
{
// A non-blank E2E_CACHE_DIR replaces cache.dir and the default; the mode stays as configured.
var shared = environment("E2E_CACHE_DIR") is { } variable && !string.IsNullOrWhiteSpace(variable) ? variable : null;
var sharedDirectory = shared is null ? null : Path.GetFullPath(Path.Combine(projectRoot, ExpandHome(shared)));
var defaultDirectory = sharedDirectory ?? Path.Combine(projectRoot, ".e2e", "cache");
if (raw is null)
{
return new CacheConfig { Mode = ci ? CacheMode.ReadOnly : CacheMode.ReadWrite, Directory = defaultDirectory };
}
if (raw.Value.ValueKind != JsonValueKind.Object)
{
throw Invalid("cache must be an object");
}
var value = raw.Value;
CheckKeys(value, "cache config", CacheKeys, UnsupportedCacheKeys, null);
CacheMode mode;
if (Property(value, "mode") is { } rawMode)
{
mode = (rawMode.ValueKind == JsonValueKind.String ? rawMode.GetString() : null) switch
{
"off" => CacheMode.Off,
"read-only" => CacheMode.ReadOnly,
"read-write" => CacheMode.ReadWrite,
_ => throw Invalid("cache.mode must be \"off\", \"read-only\", or \"read-write\""),
};
}
else
{
// An unset mode is read-write locally. CI treats recordings as untrusted input and only reads them.
mode = ci ? CacheMode.ReadOnly : CacheMode.ReadWrite;
}
var directory = defaultDirectory;
if (Property(value, "dir") is { } rawDir)
{
var dir = rawDir.ValueKind == JsonValueKind.String ? rawDir.GetString() : null;
if (string.IsNullOrWhiteSpace(dir))
{
throw Invalid("cache.dir must be a non-empty path");
}
directory = sharedDirectory ?? Path.GetFullPath(Path.Combine(projectRoot, dir));
}
var strict = false;
if (Property(value, "strict") is { } rawStrict)
{
strict = rawStrict.ValueKind switch
{
JsonValueKind.True => true,
JsonValueKind.False => false,
_ => throw Invalid("cache.strict must be a boolean"),
};
}
return new CacheConfig { Mode = mode, Directory = directory, Strict = strict };
}
/// The value with a leading ~ or ~/ read as the home directory, which no shell expands in an env file.
private static string ExpandHome(string value)
{
var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
if (value == "~")
{
return home;
}
return value.StartsWith("~/", StringComparison.Ordinal) || value.StartsWith("~" + Path.DirectorySeparatorChar, StringComparison.Ordinal)
? Path.Join(home, value[2..])
: value;
}
private static Secrets ReadSecrets(JsonElement? raw, Func environment)
{
if (raw is null)
{
return Secrets.Empty;
}
if (raw.Value.ValueKind != JsonValueKind.Object)
{
throw Invalid("secrets must be an object of secret values by name");
}
var values = new Dictionary(StringComparer.Ordinal);
foreach (var property in raw.Value.EnumerateObject())
{
if (property.Name.Trim().Length == 0)
{
throw Invalid("secret names must be non-empty");
}
}
// One E2E_SECRET_* value would replace every secret that maps to the variable, set or not.
var shared = raw.Value.EnumerateObject()
.GroupBy(property => Secrets.EnvironmentVariable(property.Name), StringComparer.Ordinal)
.FirstOrDefault(group => group.Count() > 1);
if (shared is not null)
{
var names = shared.Select(property => "\"" + property.Name + "\"").ToArray();
throw Invalid(
"secrets " + string.Join(", ", names[..^1]) + " and " + names[^1] + " share the override variable " + shared.Key
+ ", so one value set there would replace all of them; rename all but one");
}
foreach (var property in raw.Value.EnumerateObject())
{
var name = property.Name;
var variable = Secrets.EnvironmentVariable(name);
string? entry = property.Value.ValueKind switch
{
JsonValueKind.String => property.Value.GetString(),
JsonValueKind.Null => null,
_ => throw Invalid("secret \"" + name + "\" must be a non-empty string, or null to read only " + variable),
};
var value = environment(variable) ?? entry;
if (value is null)
{
throw Invalid("secret \"" + name + "\" has no value; set " + variable);
}
if (Secret.Problem(value) is { } problem)
{
throw Invalid("secret \"" + name + "\" " + problem);
}
values[name] = Secret.Create(name, value);
}
return new Secrets(values);
}
private static void CheckKeys(JsonElement value, string label, string[] allowed, string[] unsupported, Dictionary? moved, Dictionary? removed = null)
{
foreach (var property in value.EnumerateObject())
{
var key = property.Name;
if (removed is not null && removed.TryGetValue(key, out var replacement))
{
throw Invalid((label == "config" ? key : label + "." + key) + " was removed: " + replacement);
}
if (allowed.Contains(key, StringComparer.Ordinal))
{
continue;
}
var where = label == "config" ? "config key" : label + " key";
if (unsupported.Contains(key, StringComparer.Ordinal))
{
throw Invalid(where + " \"" + key + "\" is not supported by the .NET port" + (key == "executor" ? ": an executor is code, so set AgentOptions.Executor" : ""));
}
string hint;
if (moved is not null && moved.TryGetValue(key, out var movedHint))
{
hint = "; " + movedHint;
}
else
{
hint = Suggestions.DidYouMean(key, allowed);
}
throw Invalid("unknown " + where + " \"" + key + "\"" + hint);
}
}
private static JsonElement? Property(JsonElement value, string name)
{
return value.TryGetProperty(name, out var property) ? property : null;
}
private static string? OptionalString(JsonElement value, string name, string label)
{
if (Property(value, name) is not { } property)
{
return null;
}
var text = property.ValueKind == JsonValueKind.String ? property.GetString() : null;
if (string.IsNullOrWhiteSpace(text))
{
throw Invalid(label + " must be a non-empty string");
}
return text;
}
// Checks app.environment: one of test, staging, production.
private static string? ReadEnvironment(JsonElement? raw, string label)
{
if (raw is not { } value)
{
return null;
}
var text = value.ValueKind == JsonValueKind.String ? value.GetString() : null;
return text is not null && Environments.Contains(text, StringComparer.Ordinal)
? text
: throw Invalid(label + " must be one of " + string.Join(", ", Environments) + ", got " + value.GetRawText());
}
// Checks app.launchArguments: an array of strings.
private static ReadOnlyCollection? ReadLaunchArguments(JsonElement? raw, string label)
{
if (raw is not { } value)
{
return null;
}
if (value.ValueKind != JsonValueKind.Array || value.EnumerateArray().Any(argument => argument.ValueKind != JsonValueKind.String))
{
throw Invalid(label + " must be an array of strings");
}
return value.EnumerateArray().Select(argument => argument.GetString()!).ToList().AsReadOnly();
}
// Checks app.permissions: an object of permission name to grant, deny, or reset.
private static ReadOnlyDictionary? ReadPermissions(JsonElement? raw, string label)
{
if (raw is not { } value)
{
return null;
}
if (value.ValueKind != JsonValueKind.Object)
{
throw Invalid(label + " must be an object of permission name to grant, deny, or reset");
}
var permissions = new Dictionary(StringComparer.Ordinal);
foreach (var property in value.EnumerateObject())
{
var state = property.Value.ValueKind == JsonValueKind.String ? property.Value.GetString() : null;
permissions[property.Name] = state switch
{
"grant" => AppPermissionState.Grant,
"deny" => AppPermissionState.Deny,
"reset" => AppPermissionState.Reset,
_ => throw Invalid(label + "." + property.Name + " must be grant, deny, or reset, got " + property.Value.GetRawText()),
};
}
return new ReadOnlyDictionary(permissions);
}
// A string that may be empty, as upstream system and context.
private static string? AnyString(JsonElement value, string name, string label)
{
if (Property(value, name) is not { } property)
{
return null;
}
if (property.ValueKind != JsonValueKind.String)
{
throw Invalid(label + " must be a string");
}
return property.GetString();
}
private static TimeSpan Milliseconds(JsonElement root, string name, TimeSpan fallback)
{
return Property(root, name) is { } raw ? TimeSpan.FromMilliseconds(Integer(raw, name, 1, int.MaxValue)) : fallback;
}
private static int Integer(JsonElement value, string label, int min, int max)
{
if (value.ValueKind != JsonValueKind.Number || !value.TryGetInt32(out var number) || number < min || number > max)
{
throw Invalid(max == int.MaxValue
? label + " must be a positive integer"
: label + " must be an integer from " + min.ToString(CultureInfo.InvariantCulture) + " through " + max.ToString(CultureInfo.InvariantCulture));
}
return number;
}
private static bool ValidName(string name)
{
return name.Length > 0
&& name.Any(ch => ch != '.')
&& name.All(ch => char.IsAsciiLetterOrDigit(ch) || ch is '_' or '-' or '.');
}
private static ConfigurationException Invalid(string message) => new("INVALID_CONFIG", message);
}
/// One entry of targets. The port drives the web, ios, and android platforms.
public sealed class TargetConfig
{
public required string Name { get; init; }
public required string Platform { get; init; }
public required AppConfig App { get; init; }
/// This target with another app: the one resolved on the port its command was started on.
internal TargetConfig WithApp(AppConfig app) => new() { Name = Name, Platform = Platform, App = app, Trace = Trace, Video = Video };
/// Which attempts write trace.md: E2E_TRACE, else this target's trace, else the config's, else retain-on-failure.
public ResolvedRecording Trace { get; init; } = new(RecordingMode.RetainOnFailure, RecordingSource.Default);
/// Which attempts record a video: E2E_VIDEO, else this target's video, else the config's, else off.
public ResolvedRecording Video { get; init; } = new(RecordingMode.Off, RecordingSource.Default);
}
/// One kind's modes before any target speaks: the environment variable, the config root, and the default under both.
internal sealed record RunRecording(RecordingMode? Flag, RecordingMode? Config, RecordingMode Fallback)
{
/// A target's effective mode and where it came from: the variable, else the target's own, else the config root's, else the default.
public ResolvedRecording For(RecordingMode? own)
{
if (Flag is { } flag)
{
return new ResolvedRecording(flag, RecordingSource.Run);
}
if (own is { } target)
{
return new ResolvedRecording(target, RecordingSource.Target);
}
return Config is { } config ? new ResolvedRecording(config, RecordingSource.Run) : new ResolvedRecording(Fallback, RecordingSource.Default);
}
}
public sealed class AppConfig
{
///
/// Base URL for App.OpenAsync, normalized: a missing scheme becomes https://, or http:// for a loopback host.
/// Relative URLs resolve against it.
///
public string? Url { get; init; }
/// The bundle id (iOS) or package (Android) of an installed app a device target launches.
public string? BundleId { get; init; }
/// The build a device target installs, relative to the folder of e2e.config.json.
public string? AppPath { get; init; }
///
/// The command that starts the app (upstream's app.command), started before the first test and stopped when the process ends.
/// {port} in and is the port the app is served on. A
/// with port 0 asks for a free one, and needs this.
///
public CommandConfig? Command { get; init; }
/// The URL that answers once has started the app (upstream's app.readyUrl). Unset, .
public string? ReadyUrl { get; init; }
///
/// The stable identity that keys the replay cache and saved sessions (upstream's app.identity). Unset, the URL's origin and
/// path as declared, else , else . An ephemeral per-deploy origin (a PR preview) can
/// declare the identity of the app it is a deployment of, and so share its entries.
///
public string? Identity { get; init; }
///
/// What the app is: test, staging, or production (upstream's app.environment). Unset, test for a
/// loopback, .localhost, or .test host, and production for any other host. The environment always joins the
/// cache key, so an identity never bleeds entries across environments.
///
public string? Environment { get; init; }
/// Arguments every fresh launch of the app takes: process arguments on iOS, am start arguments on Android, the command line on Windows.
public IReadOnlyList? LaunchArguments { get; init; }
///
/// Permissions the app holds before every fresh launch, by permission name (upstream's app.permissions). A device target
/// takes the names of DevicePermission.
///
public IReadOnlyDictionary? Permissions { get; init; }
/// This app with another URL: the one on the port its command was started on.
internal AppConfig WithUrl(string? url) => new()
{
Url = url,
BundleId = BundleId,
AppPath = AppPath,
Identity = Identity,
Environment = Environment,
LaunchArguments = LaunchArguments,
Permissions = Permissions,
Command = Command,
ReadyUrl = ReadyUrl,
};
}
/// What a permission is set to before a launch (upstream's AppPermissionState): held, refused, or not asked for yet, so the OS asks again.
public enum AppPermissionState
{
/// The app holds the permission.
Grant,
/// The permission is refused.
Deny,
/// The permission is not asked for yet.
Reset,
}
///
/// One entry of agents. Upstream model and judge are AI SDK instances; here they are model ids
/// for , which share and .
/// Each entry starts from the defaults; none inherits another's values.
///
public sealed class AgentConfig
{
public const string DefaultProvider = "openai";
///
/// The provider values: API keys (openai, openai-responses, azure, anthropic,
/// google, bedrock, xai, openrouter, gateway, openai-compatible) and
/// subscription logins (chatgpt, copilot, grok, opencode-console).
///
public static IReadOnlyList Providers { get; } =
[
"openai", "openai-responses", "azure", "anthropic", "google", "bedrock", "xai", "openrouter", "gateway", "openai-compatible",
"chatgpt", "copilot", "grok", "opencode-console",
];
public string? Model { get; init; }
/// The model id that judges assert, waitFor, and extract. Unset, the judge is .
public string? Judge { get; init; }
/// Who serves and . One of ; openai by default.
public string Provider { get; init; } = DefaultProvider;
/// Text appended to the act rules. Judges never see it.
public string? System { get; init; }
/// Project context told to every model call, at most 16384 UTF-8 bytes.
public string? Context { get; init; }
/// The API root. Unset, the provider's own; for azure it is built from AZURE_RESOURCE_NAME.
public string? BaseUrl { get; init; }
/// The environment variable holding the key. Unset, the one the provider reads, such as ANTHROPIC_API_KEY.
public string? ApiKeyEnv { get; init; }
/// 1 through 100. Null when the entry leaves it out: applies, and Explore sets its own.
public int? MaxSteps { get; init; }
/// 1 through 100. Null when the entry leaves it out: applies, and Explore sets its own.
public int? MaxModelCalls { get; init; }
public TimeSpan JudgmentTimeout { get; init; } = E2EDefaults.JudgmentTimeout;
/// 1024 through 16777216. See .
public int MaxObservationBytes { get; init; } = E2EDefaults.MaxObservationBytes;
/// 1 through 1000000. See .
public int MaxInputTokens { get; init; } = E2EDefaults.MaxInputTokens;
/// Provider options by provider. Each client adds the entry under its own key to its request body.
public IReadOnlyDictionary? ProviderOptions { get; init; }
/// The model for , or null when no model is set.
public IAgentModel? CreateModel() => Create(Model);
/// The model for , or null when no judge is set.
public IAgentModel? CreateJudge() => Create(Judge);
/// The this entry describes, with its models created.
public AgentOptions CreateOptions()
{
return new AgentOptions
{
Model = CreateModel(),
Judge = CreateJudge(),
System = System,
Context = Context,
MaxSteps = MaxSteps,
MaxModelCalls = MaxModelCalls,
JudgmentTimeout = JudgmentTimeout,
MaxObservationBytes = MaxObservationBytes,
MaxInputTokens = MaxInputTokens,
ProviderOptions = ProviderOptions,
};
}
internal static bool IsSubscription(string provider) => provider is "chatgpt" or "copilot" or "grok" or "opencode-console";
private IAgentModel? Create(string? model)
{
if (model is null)
{
return null;
}
return Provider switch
{
"openai-responses" => new OpenAiResponsesModel(new OpenAiResponsesModelOptions
{
Model = model,
BaseUrl = BaseUrl ?? "https://api.openai.com/v1",
ApiKeyEnv = ApiKeyEnv ?? "OPENAI_API_KEY",
}),
"azure" => new OpenAiResponsesModel(new OpenAiResponsesModelOptions
{
Model = model,
BaseUrl = BaseUrl ?? AzureBaseUrl(),
Provider = "azure",
ApiKeyEnv = ApiKeyEnv ?? "AZURE_API_KEY",
ApiKeyHeader = "api-key",
QueryParameters = new Dictionary(StringComparer.Ordinal) { ["api-version"] = "v1" },
}),
"anthropic" => new AnthropicModel(new AnthropicModelOptions
{
Model = model,
BaseUrl = BaseUrl ?? "https://api.anthropic.com/v1",
ApiKeyEnv = ApiKeyEnv ?? "ANTHROPIC_API_KEY",
}),
"google" => new GoogleModel(new GoogleModelOptions
{
Model = model,
BaseUrl = BaseUrl ?? "https://generativelanguage.googleapis.com/v1beta",
ApiKeyEnv = ApiKeyEnv ?? "GOOGLE_GENERATIVE_AI_API_KEY",
}),
"bedrock" => new BedrockModel(new BedrockModelOptions
{
Model = model,
BaseUrl = BaseUrl,
ApiKeyEnv = ApiKeyEnv ?? "AWS_BEARER_TOKEN_BEDROCK",
}),
"xai" => Compatible(model, "https://api.x.ai/v1", "xai", "XAI_API_KEY", null),
"openrouter" => Compatible(model, "https://openrouter.ai/api/v1", "openrouter", "OPENROUTER_API_KEY", null),
"gateway" => Compatible(model, "https://ai-gateway.vercel.sh/v1", "gateway", "AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"),
"openai-compatible" => Compatible(model, "http://127.0.0.1:11434/v1", "openai-compatible", "LLM_API_KEY", null),
"chatgpt" => OAuth.Subscriptions.ChatGpt(model),
"copilot" => OAuth.Subscriptions.Copilot(model),
"grok" => OAuth.Subscriptions.Grok(model),
"opencode-console" => OAuth.Subscriptions.OpenCodeConsole(model),
_ => Compatible(model, "https://api.openai.com/v1", "openai", "OPENAI_API_KEY", null),
};
}
private OpenAiCompatibleModel Compatible(string model, string baseUrl, string provider, string apiKeyEnv, string? fallbackEnv)
{
return new OpenAiCompatibleModel(new OpenAiCompatibleModelOptions
{
Model = model,
BaseUrl = BaseUrl ?? baseUrl,
Provider = provider,
ApiKeyEnv = ApiKeyEnv ?? apiKeyEnv,
ApiKeyFallbackEnv = ApiKeyEnv is null ? fallbackEnv : null,
});
}
private static string AzureBaseUrl()
{
var resource = Environment.GetEnvironmentVariable("AZURE_RESOURCE_NAME");
return string.IsNullOrWhiteSpace(resource)
? throw new ConfigurationException("INVALID_CONFIG", "provider \"azure\" needs baseUrl or AZURE_RESOURCE_NAME")
: "https://" + resource.Trim() + ".openai.azure.com/openai/v1";
}
}
public sealed class CacheConfig
{
/// Unset, it is , or in CI.
public CacheMode Mode { get; init; } = CacheMode.ReadWrite;
/// Absolute cache directory. cache.dir resolves against the config directory; the default is .e2e/cache. A non-blank E2E_CACHE_DIR replaces both.
public string Directory { get; init; } = Path.Combine(".e2e", "cache");
/// When true, a recording that no longer matches fails with REPLAY_STALE instead of running live.
public bool Strict { get; init; }
}
/// How the replay cache is used.
public enum CacheMode
{
/// Never read or write recordings.
Off,
/// Replay recordings, never write or delete them.
ReadOnly,
/// Replay recordings, record verified acts, and delete unverified ones.
ReadWrite,
}