// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Text.RegularExpressions;
namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe;
///
/// Tests for the --server dotnettestcli pipe protocol covering the handshake and protocol
/// version negotiation. This PR installs the no-op output device under pipe mode and bumps the
/// advertised protocol to 1.1.0; these tests lock down that behavior (and the remaining session
/// event / stdout baselines) so further phases such as live log and stream forwarding can be
/// reviewed as targeted diffs against these assertions.
///
/// Tracks dotnet/sdk#51615 and the
/// related microsoft/testfx#7161.
///
///
[TestClass]
public class DotnetTestPipeBaselineTests : AcceptanceTestBase
{
private const string AssetName = "DotnetTestPipeBaselineTest";
private static readonly Regex BannerRegex = new(@"Microsoft\.Testing\.Platform v.+ \[.+\]", RegexOptions.Compiled);
public TestContext TestContext { get; set; } = null!;
// The test host (testfx) now advertises protocol 1.0.0 through 1.4.0. The 1.1.0 bump
// signalled that TerminalOutputDevice is no longer plugged in under the pipe protocol (microsoft/testfx#7161
// and dotnet/sdk#51615); 1.2.0 added AzureDevOpsLogMessage forwarding; 1.3.0 added generic DisplayMessage
// (warning/error) forwarding; 1.4.0 added the reverse server-control channel (server-initiated cancellation).
// This mirrors ProtocolConstants.SupportedVersions on the host side.
private const string HostAdvertisedProtocolVersions = "1.0.0;1.1.0;1.2.0;1.3.0;1.4.0";
[TestMethod]
public async Task DotnetTestPipe_TestAppAdvertisesAllSupportedVersions_NegotiatesDownToV100WithOldSdk()
{
var testHost = TestInfrastructure.TestHost.LocateFrom(
AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent);
// The fake SDK advertises only 1.0.0 (the old-SDK behavior).
FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync(
testHost, cancellationToken: TestContext.CancellationToken);
Assert.IsNotNull(result.ReceivedHandshake, "Test app never sent a handshake message.");
Assert.IsNotNull(result.SentHandshakeReply, "Fake SDK never sent a handshake reply — check that the handshake frame was received and decoded.");
Assert.IsTrue(
result.ReceivedHandshake.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, out string? appVersions),
"Handshake from test app is missing SupportedProtocolVersions.");
Assert.AreEqual(
HostAdvertisedProtocolVersions,
appVersions,
"The test app should advertise 1.0.0, 1.1.0, 1.2.0 and 1.3.0 once the DisplayMessage forwarding bump landed.");
Assert.AreEqual(
FakeDotnetTestSdk.DefaultSupportedProtocolVersions,
result.NegotiatedProtocolVersion,
"An old SDK that only supports 1.0.0 should negotiate down to '1.0.0'.");
Assert.AreEqual(
"1",
result.ReceivedHandshake[DotnetTestPipeProtocol.HandshakeProperties.AttemptNumber],
"A non-retried test host should report attempt 1.");
}
[TestMethod]
public async Task DotnetTestPipe_InvalidAttemptNumber_FailsBeforeHandshake()
{
var testHost = TestInfrastructure.TestHost.LocateFrom(
AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent);
FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync(
testHost,
environmentVariables: new()
{
{ "TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER", "0" },
},
cancellationToken: TestContext.CancellationToken);
Assert.AreNotEqual((int)ExitCode.Success, result.TestHostResult.ExitCode);
Assert.IsNull(result.ReceivedHandshake, "An invalid attempt number must fail before a handshake is sent.");
Assert.Contains(
"TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER",
result.TestHostResult.StandardOutput + result.TestHostResult.StandardError);
}
[TestMethod]
public async Task DotnetTestPipe_TestAppNegotiatesV110WithUpToDateSdk()
{
var testHost = TestInfrastructure.TestHost.LocateFrom(
AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent);
// The fake SDK advertises up to 1.1.0, so negotiation should pick the highest mutually
// supported version: 1.1.0 (even though the host also supports 1.2.0).
FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync(
testHost,
supportedProtocolVersions: "1.0.0;1.1.0",
cancellationToken: TestContext.CancellationToken);
Assert.IsNotNull(result.ReceivedHandshake, "Test app never sent a handshake message.");
Assert.IsNotNull(result.SentHandshakeReply, "Fake SDK never sent a handshake reply.");
Assert.AreEqual(
"1.1.0",
result.NegotiatedProtocolVersion,
"When both sides advertise 1.1.0, negotiation should select the highest mutually supported version.");
}
[TestMethod]
public async Task DotnetTestPipe_EmitsTestSessionStartAndEnd()
{
var testHost = TestInfrastructure.TestHost.LocateFrom(
AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent);
FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync(
testHost, cancellationToken: TestContext.CancellationToken);
byte[] sessionEventTypes =
[
.. result.MessagesWithSerializerId(DotnetTestPipeProtocol.SerializerIds.TestSessionEvent)
.Select(m => DotnetTestPipeProtocol.DecodeTestSessionEventBody(m.Body).SessionType)
.Where(t => t.HasValue)
.Select(t => t!.Value)
];
Assert.HasCount(2, sessionEventTypes, $"Expected exactly two session events; got [{string.Join(", ", sessionEventTypes)}].");
Assert.AreEqual(DotnetTestPipeProtocol.SessionEventTypes.TestSessionStart, sessionEventTypes[0]);
Assert.AreEqual(DotnetTestPipeProtocol.SessionEventTypes.TestSessionEnd, sessionEventTypes[1]);
}
///
/// BASELINE: under --server dotnettestcli the test app today emits nothing to stdout/stderr
/// for a passing run (the MTP banner is already silenced via _isServerMode in
/// TerminalOutputDevice, and our DummyTestFramework writes nothing of its own).
/// This test pins that contract so future changes can't accidentally start leaking output that
/// would compete with what the SDK is also rendering (the original symptom behind
/// dotnet/sdk#51615 and microsoft/testfx#7161). When this test fails, evaluate whether the
/// added stdout/stderr writes should instead flow through the pipe as structured messages.
///
[TestMethod]
public async Task DotnetTestPipe_ChildEmitsNoStdoutOrStderrForPassingRun_Baseline()
{
var testHost = TestInfrastructure.TestHost.LocateFrom(
AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent);
FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync(
testHost, cancellationToken: TestContext.CancellationToken);
Assert.DoesNotMatchRegex(
BannerRegex,
result.TestHostResult.StandardOutput,
$"BASELINE: under --server dotnettestcli the test app already silences its banner. " +
$"If this fails, something started re-emitting the banner under pipe mode.{Environment.NewLine}" +
$"Captured stdout:{Environment.NewLine}{result.TestHostResult.StandardOutput}");
Assert.AreEqual(
string.Empty,
result.TestHostResult.StandardOutput.Trim(),
$"BASELINE: today the test app emits nothing to stdout under pipe mode for a no-op " +
$"DummyTestFramework run. New text leaking here likely belongs in a pipe message " +
$"(Phase 2 of dotnet/sdk#51615 adds a LogMessage channel for this).{Environment.NewLine}" +
$"Captured stdout:{Environment.NewLine}{result.TestHostResult.StandardOutput}");
Assert.AreEqual(
string.Empty,
result.TestHostResult.StandardError.Trim(),
$"BASELINE: today the test app emits nothing to stderr under pipe mode for a passing run.{Environment.NewLine}" +
$"Captured stderr:{Environment.NewLine}{result.TestHostResult.StandardError}");
}
public sealed class TestAssetFixture() : TestAssetFixtureBase()
{
private const string AssetCode = """
#file DotnetTestPipeBaselineTest.csproj
$TargetFrameworks$
enable
enable
Exe
true
preview
#file Program.cs
using Microsoft.Testing.Platform.Builder;
using Microsoft.Testing.Platform.Capabilities.TestFramework;
using Microsoft.Testing.Platform.Extensions.TestFramework;
public class Program
{
public static async Task Main(string[] args)
{
ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args);
builder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestFramework());
using ITestApplication app = await builder.BuildAsync();
return await app.RunAsync();
}
}
public class DummyTestFramework : ITestFramework
{
public string Uid => nameof(DummyTestFramework);
public string Version => "2.0.0";
public string DisplayName => nameof(DummyTestFramework);
public string Description => nameof(DummyTestFramework);
public Task IsEnabledAsync() => Task.FromResult(true);
public Task CreateTestSessionAsync(CreateTestSessionContext context)
=> Task.FromResult(new CreateTestSessionResult() { IsSuccess = true });
public Task CloseTestSessionAsync(CloseTestSessionContext context)
=> Task.FromResult(new CloseTestSessionResult() { IsSuccess = true });
public Task ExecuteRequestAsync(ExecuteRequestContext context)
{
context.Complete();
return Task.CompletedTask;
}
}
""";
public string TargetAssetPath => GetAssetPath(AssetName);
public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName,
AssetCode
.PatchTargetFrameworks(TargetFrameworks.NetCurrent)
.PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion));
}
}