// Copyright 2026 TesterArmy. // Modified by Dario Kondratiuk. // SPDX-License-Identifier: Apache-2.0 namespace E2E.NUnit.Tests.McpTestRun; /// /// describe('open_session {test}') > describe('over e2e mcp') of tests/integration/mcp-test-run.test.ts (tester-army/e2e#1315). /// All eight ported its are Partial_: seven run , the seam e2e mcp uses, and the eighth loads the assembly through SessionTests.Load; none runs the MCP server /// (the server path was checked by hand; McpSessionTest/SessionTestsTests covers the loading and the refusals, and /// McpTestOutcome/TestOutcomeTests the opening text). The first leaves out afterEach at close_session (hooks run before the /// hand-over). The second checks the session holds the failing step's line and the evidence the text prints, not the text itself. The third /// has no file:line, no target, no serial group, no several agents, and refuses a saved session before the run, where upstream also refuses the /// setup test. The fourth ("compares a screenshot against the baseline an earlier session wrote") runs a fixture twice through the runner and /// checks the first outcome fails with no stored screenshot and the second passes; it does not check the opening text or name the test /// as file:line. Three more run the same seam and leave out the text and the server too (#1410): "reports a beforeAll failure as the outcome, once" /// checks the outcome (no session, HOOK_FAILED, phase beforeAll, nothing to end), and not 0 steps ran or No screen to show; /// "fails a test on a promise it left rejected, and keeps serving" checks the failure, the close's report, and the operator's warning of the /// seam's , and not the opening text; "keeps a body its timeout cut short from driving the session" runs a fixture with a /// 5 s timeout (upstream's is 1 s, and the port's clock starts before the engine launches, so the first page must open inside it). It checks the /// TEST_TIMEOUT failure, the step count, and the screen the session observes after its body timed out, but not the opening text; NUnit awaits the body, so the outcome /// lands after it, and not at the timeout. "runs the test files as they are now when the session opens again" is Partial_: it loads the test assembly twice through /// SessionTests.Load and checks the second load is a new one, not the server's open twice (an assembly loads once per server; hardkoded/e2e-dotnet#1411). /// Not ported, with the issue that extracts what each needs: /// "records a session's agent steps for the next session and for e2e run" (describe 'the replay cache'; ported in part as `TheReplayCacheTests`, which lacks the `e2e run` half until the run seam of #1336), /// the two test.fail its ("tells a test that /// failed as test.fail expected, and one that passed unexpectedly", "does not cover a beforeAll failure with test.fail"; the port has no test.fail, /// see COMPATIBILITY.md). The unit it in mcp-test-lines.test.ts ("drafts the lines after /// a test the session ran as the ones that continue it") is McpTestLines/SessionTestLinesTests in E2E.Cli.Tests. /// public sealed class OpenSessionTestOverE2eMcpTests { [Test] public async Task Partial_Leaves_a_passing_test_on_its_final_screen_and_runs_its_afterEach_at_close_session() { var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Opens_the_browser_page), CancellationToken.None); Assert.That(outcome, Is.Not.Null); await using var closing = outcome!; var session = outcome!.Session!; Assert.That(outcome.Failure, Is.Null); await Expect.That(session.Screen.GetByLabel("Loads")).ToBeVisibleAsync(); } [Test] public async Task Partial_Opens_a_failing_test_with_its_code_line_code_frame_and_nearest_nodes_on_the_screen_it_failed_on() { var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Fails_on_the_browser_page), CancellationToken.None); Assert.That(outcome, Is.Not.Null); await using var closing = outcome!; var session = outcome!.Session!; Assert.That(outcome.Failure, Is.Not.Null); Assert.That(outcome.Failure!.Message, Does.Contain("the count")); // What the opening text prints: the line of the failing step, and the screen the failure landed on. Assert.That(session.StepRecords.Last(step => step.Status != StepStatus.Passed).Source?.File, Does.EndWith("OpenSessionTestOverE2eMcpTests.cs")); Assert.That(session.Evidence?.Url, Does.Contain("/browser")); await Expect.That(session.Screen.GetByLabel("Loads")).ToBeVisibleAsync(); } [Test] public void Partial_Refuses_a_name_that_selects_no_test_several_or_one_a_session_cannot_run_alone_before_anything_boots() { var runner = new NUnitSessionTestRunner(); var assembly = typeof(SessionFixture).Assembly; Assert.That(runner.Select(assembly, "No.Such.Test"), Is.Empty); var several = runner.Select(assembly, typeof(SessionFixture).FullName!); Assert.That(several.Select(test => test.FullName), Is.EquivalentTo([typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Opens_the_browser_page), typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Fails_on_the_browser_page), typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Starts_from_a_saved_session)])); var saved = runner.Select(assembly, typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Starts_from_a_saved_session)); Assert.That(saved, Has.Count.EqualTo(1)); Assert.That(saved[0].Unsupported, Is.EqualTo("test \"" + typeof(SessionFixture).FullName + ".Starts_from_a_saved_session\" starts from session \"member\", which a setup test saves first; a session does not run setup tests yet, so run it with dotnet test")); Assert.That(runner.Select(assembly, typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Opens_the_browser_page))[0].Unsupported, Is.Null); } [Test] [NonParallelizable] public async Task Partial_Compares_a_screenshot_against_the_baseline_an_earlier_session_wrote() { var name = typeof(ScreenshotSessionFixture).FullName + "." + nameof(ScreenshotSessionFixture.Matches_its_screenshot); var snapshots = ScreenshotSessionFixture.SnapshotsDirectory(); var ci = Environment.GetEnvironmentVariable("CI"); var update = Environment.GetEnvironmentVariable("E2E_UPDATE_SNAPSHOTS"); // The directory belongs to this fixture alone: nothing else stores screenshots beside this file. if (Directory.Exists(snapshots)) { Directory.Delete(snapshots, recursive: true); } // As upstream's server runs with CI empty: a CI run writes no baseline, and an update run writes it and passes. Environment.SetEnvironmentVariable("CI", "0"); Environment.SetEnvironmentVariable("E2E_UPDATE_SNAPSHOTS", null); try { var first = await RunAsync(name); Assert.That(first.Failure, Is.Not.Null); Assert.That(first.Failure!.Message, Does.Contain("no stored screenshot")); // Upstream's store of written screenshots is per store, so the second session starts with none; the port's is per process. ScreenshotContext.ForgetWrittenUnder(snapshots); var second = await RunAsync(name); Assert.That(second.Failure, Is.Null); } finally { Environment.SetEnvironmentVariable("CI", ci); Environment.SetEnvironmentVariable("E2E_UPDATE_SNAPSHOTS", update); if (Directory.Exists(snapshots)) { Directory.Delete(snapshots, recursive: true); } } } [Test] public async Task Partial_Reports_a_beforeAll_failure_as_the_outcome_once() { var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, typeof(HooksSessionFixture).FullName + "." + nameof(HooksSessionFixture.Needs_the_database), CancellationToken.None); // The test never opened the app, and the outcome is the hook's: no session to show, and nothing for the close to repeat. Assert.That(outcome, Is.Not.Null); Assert.That(outcome!.Session, Is.Null); Assert.That(outcome.Phase, Is.EqualTo(RunPhase.BeforeAll)); Assert.That(outcome.Failure, Is.TypeOf()); Assert.That(((TestException)outcome.Failure!).Code, Is.EqualTo("HOOK_FAILED")); Assert.That(outcome.Failure!.Message, Is.EqualTo("beforeAll failed: database is down")); Assert.That(outcome.Run, Is.Null); } [Test] [NonParallelizable] public async Task Partial_Fails_a_test_on_a_promise_it_left_rejected_and_keeps_serving() { var runner = new NUnitSessionTestRunner(); var assembly = typeof(SessionFixture).Assembly; var warnings = new System.Collections.Concurrent.ConcurrentQueue(); var warn = StrayRejections.Warn; StrayRejections.Warn = warnings.Enqueue; try { var forgot = await runner.RunAsync(assembly, typeof(SloppySessionFixture).FullName + "." + nameof(SloppySessionFixture.Forgets_an_await), CancellationToken.None); Assert.That(forgot, Is.Not.Null); await using (forgot!.Session) { Assert.That(forgot.Failure?.Message, Is.EqualTo("seed endpoint returned 500")); Assert.That(forgot.Run!.End(), Is.Empty); } // One that rejects after the body ended is the test's too, reported at close. var left = await runner.RunAsync(assembly, typeof(SloppySessionFixture).FullName + "." + nameof(SloppySessionFixture.Leaves_a_promise_behind), CancellationToken.None); Assert.That(left, Is.Not.Null); Assert.That(left!.Failure, Is.Null); SloppySessionFixture.LeftRelease.SetResult(); await Task.Delay(300); await using (left.Session) { var closed = left.Run!.End(); Assert.That(closed.Select(cause => (cause as TestException)?.Code + ": " + cause.Message), Is.EqualTo(["UNHANDLED_REJECTION: a promise rejected after the test's body ended, most likely left by \"" + left.TestName + "\": late failure"])); } // One that rejects once its session closed is a warning for the operator, not a crash. var early = await runner.RunAsync(assembly, typeof(SloppySessionFixture).FullName + "." + nameof(SloppySessionFixture.Leaves_a_promise_for_after_the_close), CancellationToken.None); Assert.That(early, Is.Not.Null); Assert.That(early!.Failure, Is.Null); await using (early.Session) { Assert.That(early.Run!.End(), Is.Empty); } SloppySessionFixture.EarlyRelease.SetResult(); // .NET reports the faulted task when it is collected: look until it lands, as the host does for a few seconds after a close. for (var look = 0; look < 40 && warnings.IsEmpty; look++) { await Task.Delay(250); StrayRejections.Collect(); } Assert.That(warnings, Does.Contain("a promise rejected after its session closed, most likely left by test \"" + early.TestName + "\": later failure")); var after = await runner.RunAsync(assembly, typeof(SessionFixture).FullName + "." + nameof(SessionFixture.Opens_the_browser_page), CancellationToken.None); Assert.That(after, Is.Not.Null); await using (after!.Session) { Assert.That(after.Failure, Is.Null); } } finally { StrayRejections.Warn = warn; } } /// Runs the named test as a session and disposes its session; what is left is the outcome's name and failure. private static async Task<(string TestName, Exception? Failure)> RunAsync(string name) { var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, name, CancellationToken.None); Assert.That(outcome, Is.Not.Null); await using var closing = outcome!; var session = outcome!.Session!; return (outcome.TestName, outcome.Failure); } [Test] public void Partial_Runs_the_test_files_as_they_are_now_when_the_session_opens_again() { // The test assembly is collected again on every open, from the bytes on disk now: a rebuilt test replaces the file even while a session holds the old one. var directory = Path.Combine(Path.GetTempPath(), "e2e-reload-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); try { var source = Path.GetDirectoryName(typeof(SessionFixture).Assembly.Location)!; foreach (var file in Directory.EnumerateFiles(source, "E2E.NUnit.Tests.*")) { File.Copy(file, Path.Combine(directory, Path.GetFileName(file))); } var path = Path.Combine(directory, "E2E.NUnit.Tests.dll"); using var first = E2E.Cli.Mcp.SessionTests.Load(path); File.Delete(path); File.Copy(Path.Combine(source, "E2E.NUnit.Tests.dll"), path); using var second = E2E.Cli.Mcp.SessionTests.Load(path); Assert.That(second.Tests, Is.Not.SameAs(first.Tests)); Assert.That(second.Tests.GetType(typeof(SessionFixture).FullName!), Is.Not.Null); } finally { Directory.Delete(directory, recursive: true); } } [Test] public void Selects_a_test_by_file_and_line_and_refuses_a_line_inside_its_body() { using var loaded = E2E.Cli.Mcp.SessionTests.Load(typeof(SessionFixture).Assembly.Location); var lines = File.ReadAllLines(SourcePath()); int LineOf(string fragment) => Array.FindIndex(lines, line => line.Contains(fragment, StringComparison.Ordinal)) + 1; var opens = LineOf("public async Task Opens_the_" + "browser_page() =>"); var fails = LineOf("public async Task Fails_on_" + "the_browser_page()"); var expected = typeof(SessionFixture).FullName + "."; // The attribute line and the signature line both name the test. Assert.That(E2E.Cli.Mcp.SessionTests.SelectOne(loaded.Runner, loaded.Tests, "McpTestRun/OpenSessionTestOverE2eMcpTests.cs:" + opens, "tests.dll"), Is.EqualTo(expected + "Opens_the_browser_page")); Assert.That(E2E.Cli.Mcp.SessionTests.SelectOne(loaded.Runner, loaded.Tests, "OpenSessionTestOverE2eMcpTests.cs:" + (opens - 1), "tests.dll"), Is.EqualTo(expected + "Opens_the_browser_page")); Assert.That(E2E.Cli.Mcp.SessionTests.SelectOne(loaded.Runner, loaded.Tests, "OpenSessionTestOverE2eMcpTests.cs:" + fails, "tests.dll"), Is.EqualTo(expected + "Fails_on_the_browser_page")); var inside = Assert.Throws(() => E2E.Cli.Mcp.SessionTests.SelectOne(loaded.Runner, loaded.Tests, "OpenSessionTestOverE2eMcpTests.cs:" + (fails + 2), "tests.dll")); Assert.That(inside!.Code, Is.EqualTo("NO_TESTS")); Assert.That(inside.Message, Does.Contain("names no test (declared at lines ")); } private static string SourcePath([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path; [Test] public async Task Partial_Keeps_a_body_its_timeout_cut_short_from_driving_the_session() { var name = typeof(TimedOutSessionFixture).FullName + "." + nameof(TimedOutSessionFixture.Outlives_its_timeout); var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, name, CancellationToken.None); Assert.That(outcome, Is.Not.Null); await using var closing = outcome!; var session = outcome!.Session!; Assert.That(outcome.Failure, Is.TypeOf()); Assert.That(((TestException)outcome.Failure!).Code, Is.EqualTo("TEST_TIMEOUT")); Assert.That(outcome.Failure!.Message, Is.EqualTo("test timed out after 5000 ms in phase body")); Assert.That(outcome.Phase, Is.EqualTo(RunPhase.Body)); // Upstream's "1 step ran": the step its timeout cut short (app.open('/about')) is not recorded. Assert.That(session.StepRecords, Has.Count.EqualTo(1)); // Past the time the body's last line would have run: the session still observes the screen the body left, not the page that line opens. await Expect.That(session.Screen.GetByRole("heading", "Playground")).ToBeVisibleAsync(); await Expect.That(session.Screen.GetByRole("heading", "About")).Not.ToBeVisibleAsync(); } [Test] public async Task Finds_no_session_for_a_name_that_selects_no_test() { var outcome = await new NUnitSessionTestRunner().RunAsync(typeof(SessionFixture).Assembly, "No.Such.Test", CancellationToken.None); Assert.That(outcome, Is.Null); } } /// The tests runs. [Explicit("Run by OpenSessionTestOverE2eMcpTests")] public sealed class SessionFixture : E2ETest { private static readonly E2EConfig FixtureConfig = E2EConfig.Parse("""{ "cache": { "mode": "off" } }""", SourceDirectory(), _ => null); // The project root is where the test sources are, so a step names the line it was called from. private static string SourceDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => Path.GetDirectoryName(path)!; protected override E2EConfig Config => FixtureConfig; protected override string? BaseUrl => E2E.Playground.Testbed.Url; [Test] public async Task Opens_the_browser_page() => await App.OpenAsync("/browser"); [Test] [E2ESession("member")] public void Starts_from_a_saved_session() { } [Test] public async Task Fails_on_the_browser_page() { await App.OpenAsync("/browser"); await Expect.That(Screen.GetByLabel("Loads")).ToHaveTextAsync("the count", timeout: TimeSpan.FromMilliseconds(200)); } } /// The test runs for a body its timeout cut short. [Explicit("Run by OpenSessionTestOverE2eMcpTests")] public sealed class TimedOutSessionFixture : E2ETest { private static readonly E2EConfig FixtureConfig = E2EConfig.Parse("""{ "cache": { "mode": "off" } }""", SourceDirectory(), _ => null); private static string SourceDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => Path.GetDirectoryName(path)!; protected override E2EConfig Config => FixtureConfig; protected override string? BaseUrl => E2E.Playground.Testbed.Url; // The port's clock starts before the engine launches, so the timeout is longer than upstream's 1 s: the first page still opens inside it. protected override TimeSpan TestTimeout => TimeSpan.FromSeconds(5); [Test] public async Task Outlives_its_timeout() { await App.OpenAsync("/"); await Task.Delay(TimeSpan.FromSeconds(6)); await App.OpenAsync("/about"); } } /// The test runs for the screenshot baseline. [Explicit("Run by OpenSessionTestOverE2eMcpTests")] public sealed class ScreenshotSessionFixture : E2ETest { private static readonly E2EConfig FixtureConfig = E2EConfig.Parse("""{ "cache": { "mode": "off" } }""", SourceDirectory(), _ => null); private static string SourceDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => Path.GetDirectoryName(path)!; protected override E2EConfig Config => FixtureConfig; protected override string? BaseUrl => E2E.Playground.Testbed.Url; /// Where the stored screenshots of this file live: <test file>-snapshots. public static string SnapshotsDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => path + "-snapshots"; [Test] public async Task Matches_its_screenshot() { await App.OpenAsync("/swatches"); await Expect.That(Screen.GetByTestId("far")).ToHaveScreenshotAsync("far.png"); } } /// The tests of sloppy.e2e.ts that runs, but the one that outlives its timeout (). [Explicit("Run by OpenSessionTestOverE2eMcpTests")] public sealed class SloppySessionFixture : E2ETest { private static readonly E2EConfig FixtureConfig = E2EConfig.Parse("""{ "cache": { "mode": "off" } }""", SourceDirectory(), _ => null); private static string SourceDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => Path.GetDirectoryName(path)!; protected override E2EConfig Config => FixtureConfig; protected override string? BaseUrl => E2E.Playground.Testbed.Url; private static async Task SeedAsync() { await Task.Yield(); throw new InvalidOperationException("seed endpoint returned 500"); } [Test] public async Task Forgets_an_await() { await App.OpenAsync("/browser"); _ = SeedAsync(); await Task.Delay(100); await Expect.That(Screen.GetByLabel("Loads")).ToBeVisibleAsync(); } /// /// Released by the test once the run of "leaves a promise behind" has returned, so its rejection lands after the turn and not during it: /// a timer of a fixed length races the run's own work on a slow runner. /// public static readonly TaskCompletionSource LeftRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); /// Released by the test once the session of "leaves a promise for after the close" has closed. public static readonly TaskCompletionSource EarlyRelease = new(TaskCreationOptions.RunContinuationsAsynchronously); [Test] public async Task Leaves_a_promise_behind() { await App.OpenAsync("/browser"); _ = Task.Run(async () => { await LeftRelease.Task; throw new InvalidOperationException("late failure"); }); } [Test] public async Task Leaves_a_promise_for_after_the_close() { await App.OpenAsync("/browser"); _ = Task.Run(async () => { await EarlyRelease.Task; throw new InvalidOperationException("later failure"); }); } } /// The test of hooks.e2e.ts that runs: its beforeAll fails. [Explicit("Run by OpenSessionTestOverE2eMcpTests")] public sealed class HooksSessionFixture : E2ETest { private static readonly E2EConfig FixtureConfig = E2EConfig.Parse("""{ "cache": { "mode": "off" } }""", SourceDirectory(), _ => null); private static string SourceDirectory([System.Runtime.CompilerServices.CallerFilePath] string path = "") => Path.GetDirectoryName(path)!; protected override E2EConfig Config => FixtureConfig; protected override string? BaseUrl => E2E.Playground.Testbed.Url; [OneTimeSetUp] public void BeforeAll() => throw new InvalidOperationException("database is down"); [Test] public async Task Needs_the_database() => await App.OpenAsync("/browser"); }