// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; using SL = Microsoft.Build.Logging.StructuredLogger; namespace MSTest.Acceptance.IntegrationTests; [TestClass] public sealed class SdkTests : AcceptanceTestBase { private const string AssetName = "MSTestSdk"; private const string SingleTestSourceCode = """ #file MSTestSdk.csproj true $TargetFramework$ x64 $(NoWarn);NU1507 $ExtraProperties$ #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MSTestSdkTest { [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod1() { } } } """; private const string SingleTestSourceCodeVSTest = SingleTestSourceCode + """ #file global.json { "test": { "runner": "VSTest" } } """; public TestContext TestContext { get; set; } [TestMethod] [DataRow(false)] [DataRow(true)] public async Task DotnetRun_WithCoverageSettings_PreservesInvocationWorkingDirectory(bool absoluteSettingsPath) { const string directoryTestSource = """ #file DirectoryTests.cs using System; using System.IO; using System.Runtime.CompilerServices; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MSTestSdkTest { public static class StartupDirectory { // MSTest switches directories for test execution; capture the process startup directory first. [ModuleInitializer] public static void CaptureChildWorkingDirectory() { if (Array.IndexOf(Environment.GetCommandLineArgs(), "--internal-testhostcontroller-pid") >= 0) { File.WriteAllText(Environment.GetEnvironmentVariable("WORKING_DIRECTORY_MARKER")!, Environment.CurrentDirectory); } } } [TestClass] public class DirectoryTests { [TestMethod] public void ControllerChildWorkingDirectory() { Assert.Contains("--internal-testhostcontroller-pid", Environment.GetCommandLineArgs()); Assert.IsNull(Environment.GetEnvironmentVariable("DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY")); } } } #file test.runsettings True False True False False """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, (SingleTestSourceCode + directoryTestSource) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "true")); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build \"{testAsset.TargetAssetPath}\" -c Release", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string settingsPath = absoluteSettingsPath ? Path.Combine(testAsset.TargetAssetPath, "test.runsettings") : "test.runsettings"; string markerPath = Path.Combine(testAsset.TargetAssetPath, "working-directory.txt"); string resultsDirectory = Path.Combine(testAsset.TargetAssetPath, "TestResults"); DotnetMuxerResult runResult = await DotnetCli.RunAsync( $"run --no-build -c Release -f {TargetFrameworks.NetCurrent} -- --coverage --coverage-settings \"{settingsPath}\" --coverage-output-format cobertura --coverage-output coverage.cobertura.xml --results-directory \"{resultsDirectory}\" --no-ansi --progress off", workingDirectory: testAsset.TargetAssetPath, environmentVariables: new() { ["DOTNET_CLI_TEST_COMMAND_WORKING_DIRECTORY"] = null, ["WORKING_DIRECTORY_MARKER"] = markerPath, }, cancellationToken: TestContext.CancellationToken); runResult.AssertExitCodeIs(0); const string expectedSummary = """ total: 2 failed: 0 succeeded: 2 skipped: 0 """; Assert.Contains(expectedSummary.ReplaceLineEndings(), runResult.StandardOutput.ReplaceLineEndings(), runResult.ToString()); Assert.AreEqual(testAsset.TargetAssetPath, await File.ReadAllTextAsync(markerPath, TestContext.CancellationToken)); var coverage = System.Xml.Linq.XDocument.Load(Path.Combine(resultsDirectory, "coverage.cobertura.xml")); Assert.AreEqual("coverage", coverage.Root?.Name.LocalName); Assert.IsTrue(coverage.Descendants("class").Any(element => element.Attribute("name")?.Value == "MSTestSdkTest.DirectoryTests" && element.Descendants("line").Any(line => int.Parse(line.Attribute("hits")!.Value, CultureInfo.InvariantCulture) > 0))); } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task RunTests_With_VSTest(string multiTfm, BuildConfiguration buildConfiguration) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCodeVSTest .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", "true")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {testAsset.TargetAssetPath}", workingDirectory: testAsset.TargetAssetPath, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); // 'dotnet test' runs the target frameworks in parallel and writes each framework's summary as two separate // writes: the 'Passed! - Failed: ...' counts, then the ' - MSTestSdk.dll ()' suffix. Those writes // interleave, so the two halves of one framework's summary are not guaranteed to share an output line. Assert // the two independent facts instead: every expected framework ran, and every framework reported a clean result. List expectedTargetFrameworks = ["net10.0"]; #if !SKIP_INTERMEDIATE_TARGET_FRAMEWORKS expectedTargetFrameworks.Add("net8.0"); #endif if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { expectedTargetFrameworks.Add("net462"); } foreach (string targetFramework in expectedTargetFrameworks) { compilationResult.AssertOutputContains($" - MSTestSdk.dll ({targetFramework})"); } compilationResult.AssertOutputMatchesRegexTimes("Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1", expectedTargetFrameworks.Count); compilationResult.AssertOutputDoesNotContain("Failed!"); } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task RunTests_With_MSTestRunner_DotnetTest(string multiTfm, BuildConfiguration buildConfiguration) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", string.Empty)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} --project {testAsset.TargetAssetPath} --progress off --no-ansi", workingDirectory: testAsset.TargetAssetPath, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); compilationResult.AssertOutputMatchesRegex(@"MSTestSdk.*? \(net10\.0\|x64\) passed"); #if !SKIP_INTERMEDIATE_TARGET_FRAMEWORKS compilationResult.AssertOutputMatchesRegex(@"MSTestSdk.*? \(net8\.0\|x64\) passed"); #endif if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { compilationResult.AssertOutputMatchesRegex(@"MSTestSdk.*? \(net48\|x64\) passed"); } } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task RunTests_With_MSTestRunner_Standalone(string multiTfm, BuildConfiguration buildConfiguration) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", string.Empty)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } [TestMethod] [DataRow("MTP", "Unset", "", "cs;de;es;fr;it;ja;ko;pl;pt-BR;ru;tr;zh-Hans;zh-Hant")] [DataRow("MTP", "Empty", "", "cs;de;es;fr;it;ja;ko;pl;pt-BR;ru;tr;zh-Hans;zh-Hant")] [DataRow("MTP", "SingleLanguage", "fr", "fr")] [DataRow("MTP", "CaseInsensitiveLanguage", "FR", "fr")] [DataRow("MTP", "MultipleLanguages", "fr;ja", "fr;ja")] [DataRow("MTP", "SpecificCultureDoesNotMatchParent", "fr-FR", "")] [DataRow("MTP", "ParentCultureDoesNotMatchSpecific", "pt", "")] [DataRow("MTP", "NeutralLanguageOnly", "en-US", "cs;de;es;fr;it;ja;ko;pl;pt-BR;ru;tr;zh-Hans;zh-Hant")] [DataRow("MTP", "NeutralLanguageFilter", "en-USen-US", "")] [DataRow("VSTest", "Unset", "", "fr")] [DataRow("VSTest", "Empty", "", "fr")] [DataRow("VSTest", "SingleLanguage", "fr", "fr")] [DataRow("VSTest", "CaseInsensitiveLanguage", "FR", "fr")] [DataRow("VSTest", "MultipleLanguages", "fr;ja", "fr")] [DataRow("VSTest", "DifferentLanguage", "ja", "")] [DataRow("VSTest", "SpecificCultureDoesNotMatchParent", "fr-FR", "")] [DataRow("VSTest", "NeutralLanguageOnly", "en-US", "fr")] [DataRow("VSTest", "NeutralLanguageFilter", "en-USen-US", "")] public async Task SatelliteResourceLanguages_FiltersAdapterResources(string runner, string scenario, string properties, string expectedCultures) { string assetName = $"{AssetName}{runner}Satellites{scenario}"; string runnerProperties = runner == "VSTest" ? "true" : string.Empty; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( assetName, SingleTestSourceCode .PatchCodeWithReplace("MSTestSdk.csproj", $"{assetName}.csproj") .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", runnerProperties + properties)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", environmentVariables: new() { ["DOTNET_CLI_UI_LANGUAGE"] = "fr" }, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); string outputDirectory = Path.Combine(testAsset.TargetAssetPath, "bin", BuildConfiguration.Release.ToString(), TargetFrameworks.NetCurrent); string[] expected = expectedCultures.Split(';', StringSplitOptions.RemoveEmptyEntries); AssertSatelliteCultures(outputDirectory, "MSTest.TestAdapter.resources.dll", expected); AssertSatelliteCultures(outputDirectory, "MSTestAdapter.PlatformServices.resources.dll", expected); } [TestMethod] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "Classic UWP package assets are produced only on Windows.")] [DataRow("Unset", "", "fr;fr")] [DataRow("Empty", "", "fr;fr")] [DataRow("MatchingLanguage", "fr", "fr;fr")] [DataRow("CaseInsensitiveLanguage", "FR", "fr;fr")] [DataRow("DifferentLanguage", "ja", "")] [DataRow("SpecificCultureDoesNotMatchParent", "fr-FR", "")] public async Task SatelliteResourceLanguages_FiltersClassicUwpAdapterResources(string scenario, string properties, string expectedCultures) { const string Source = """ #file ClassicUwpSatelliteResources.csproj $TargetFramework$ $Properties$ """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( $"{AssetName}ClassicUwpSatellites{scenario}", Source .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$Properties$", properties)); DotnetMuxerResult result = await DotnetCli.RunAsync( $"msbuild {testAsset.TargetAssetPath} -restore -t:PrintSatelliteCultures", environmentVariables: new() { ["DOTNET_CLI_UI_LANGUAGE"] = "fr" }, cancellationToken: TestContext.CancellationToken); result.AssertExitCodeIs(0); result.AssertOutputContains($"SatelliteCultures=[{expectedCultures}]"); } [TestMethod] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "Classic UWP package assets are produced only on Windows.")] public async Task SatelliteResourceLanguages_FiltersClassicUwpMtpAdapterResources() { const string Source = """ #file ClassicUwpMtpSatelliteResources.csproj $TargetFramework$ true fr;ja """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( $"{AssetName}ClassicUwpMtpSatellites", Source .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent)); DotnetMuxerResult result = await DotnetCli.RunAsync( $"msbuild {testAsset.TargetAssetPath} -restore -t:PrintSatelliteCultures", environmentVariables: new() { ["DOTNET_CLI_UI_LANGUAGE"] = "de" }, cancellationToken: TestContext.CancellationToken); result.AssertExitCodeIs(0); result.AssertOutputContains("SatelliteCultures=[fr;fr;ja;ja]"); } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task RunTests_With_CentralPackageManagement_Standalone(string multiTfm, BuildConfiguration buildConfiguration) { // Exercise CPM with CentralPackageVersionOverrideEnabled=false to ensure MSTest.Sdk // does not rely on the (then-forbidden) VersionOverride attribute and instead injects // PackageVersion items for its implicit references. const string CpmSourceCode = SingleTestSourceCode + """ #file Directory.Packages.props true false """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, CpmSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", string.Empty)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } public static IEnumerable> RunTests_With_MSTestRunner_Standalone_Plus_Extensions_Data() { foreach ((string MultiTfm, BuildConfiguration BuildConfiguration) buildConfig in GetBuildMatrixMultiTfmFoldedBuildConfiguration()) { yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--coverage", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--retry-failed-tests 3", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-trx", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--crashdump", "--hangdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--hangdump", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-azdo", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-gh", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-ctrf", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-html", "--crashdump")); yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-junit", "--crashdump")); // OpenTelemetry is API-only (no CLI flag); the enable property only opts the package into the build. // We pass an empty enable arg to validate the package restores and the test host runs cleanly, and we // assert that an unrelated extension's CLI arg ('--crashdump') is rejected to prove no other extensions leaked in. yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", string.Empty, "--crashdump")); } } [TestMethod] [DynamicData(nameof(RunTests_With_MSTestRunner_Standalone_Plus_Extensions_Data))] public async Task RunTests_With_MSTestRunner_Standalone_Selectively_Enabled_Extensions(string multiTfm, BuildConfiguration buildConfiguration, string msbuildExtensionEnableFragment, string enableCommandLineArg, string invalidCommandLineArg) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", msbuildExtensionEnableFragment)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); TestHostResult testHostResult = await testHost.ExecuteAsync(command: enableCommandLineArg, cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); testHostResult = await testHost.ExecuteAsync(command: invalidCommandLineArg, cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); } } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(string multiTfm, BuildConfiguration buildConfiguration) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", "AllMicrosoft"), addPublicFeeds: true); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo --report-gh --report-html", cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } public static IEnumerable<(string MultiTfm, BuildConfiguration BuildConfiguration, bool EnableDefaultExtensions)> RunTests_With_MSTestRunner_Standalone_Default_Extensions_Data() { foreach ((string MultiTfm, BuildConfiguration BuildConfiguration) buildConfig in GetBuildMatrixMultiTfmFoldedBuildConfiguration()) { yield return new(buildConfig.MultiTfm, buildConfig.BuildConfiguration, true); yield return new(buildConfig.MultiTfm, buildConfig.BuildConfiguration, false); } } [TestMethod] [DynamicData(nameof(RunTests_With_MSTestRunner_Standalone_Default_Extensions_Data))] public async Task RunTests_With_MSTestRunner_Standalone_Enable_Default_Extensions(string multiTfm, BuildConfiguration buildConfiguration, bool enableDefaultExtensions) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", enableDefaultExtensions ? string.Empty : "None")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --report-trx", cancellationToken: TestContext.CancellationToken); if (enableDefaultExtensions) { testHostResult.AssertOutputContainsSummary(0, 1, 0); } else { testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); } } } [TestMethod] [DynamicData(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration), typeof(AcceptanceTestBase))] public async Task Invalid_TestingProfile_Name_Should_Fail(string multiTfm, BuildConfiguration buildConfiguration) { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", "WrongName")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {buildConfiguration} {testAsset.TargetAssetPath}", failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(1); compilationResult.AssertOutputContains("Invalid value for property TestingExtensionsProfile. Valid values are 'Default', 'AllMicrosoft' and 'None'."); } [TestMethod] [OSCondition(ConditionMode.Exclude, OperatingSystems.OSX)] public async Task NativeAot_Smoke_Test() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ true false false """), addPublicFeeds: true); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"publish -r {RID} -f {TargetFrameworks.NetCurrent} {testAsset.TargetAssetPath}", warnAsError: true, cancellationToken: TestContext.CancellationToken); compilationResult.AssertOutputContains("Generating native code"); foreach (string fileName in TrimAndAotAssertions.MSTestOwnedSourceFiles) { compilationResult.AssertOutputDoesNotContain(fileName); } var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, verb: Verb.publish); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.Success); testHostResult.AssertOutputContainsSummary(0, 1, 0); } [TestMethod] public async Task PublishReadyToRun_Smoke_Test() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ true false false """), addPublicFeeds: true); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"publish -r {RID} -f {TargetFrameworks.NetCurrent} {testAsset.TargetAssetPath}", warnAsError: true, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, RID, Verb.publish); string publishedAssemblyPath = Path.Combine(testHost.DirectoryName, $"{AssetName}.dll"); ReadyToRunAssertions.AssertIsReadyToRunImage(publishedAssemblyPath); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.Success); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); } [TestMethod] public async Task SettingIsTestApplicationToFalseReducesAddedExtensionsAndMakesProjectNotExecutable() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCodeVSTest .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "false")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test {testAsset.TargetAssetPath}", workingDirectory: testAsset.TargetAssetPath, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); SL.Build binLog = BinlogReader.Read(compilationResult.BinlogPath!); SL.Task cscTask = binLog.FindChildrenRecursive(task => task.Name == "Csc").Single(); SL.Item[] references = [.. cscTask.FindChildrenRecursive(p => p.Name == "References").Single().Children.OfType()]; // Ensure that MSTest.Framework is referenced Assert.Contains(r => r.Text.EndsWith("MSTest.TestFramework.dll", StringComparison.OrdinalIgnoreCase), references); Assert.Contains(r => r.Text.EndsWith("MSTest.TestFramework.Extensions.dll", StringComparison.OrdinalIgnoreCase), references); // No adapter, no extensions, no vstest sdk Assert.DoesNotContain(r => r.Text.EndsWith("MSTest.TestAdapter.dll", StringComparison.OrdinalIgnoreCase), references); Assert.DoesNotContain(r => r.Text.Contains("Microsoft.Testing.Extensions.", StringComparison.OrdinalIgnoreCase), references); // It's not an executable Assert.DoesNotContain(p => p.Value == "Exe", binLog.FindChildrenRecursive(p => p.Name == "OutputType")); } [TestMethod] public async Task TestApplicationRunsTestsFromReferencedMSTestSdkTestLibrary() { const string TestApplicationWithReferencedTestLibrary = """ #file MSTestSdkTestApplication.csproj true $TargetFramework$ x64 #file ReferencedLibraryTests.cs using Microsoft.VisualStudio.TestTools.UnitTesting; using TestLibrary; namespace MSTestSdkTestApplication; [TestClass] public sealed class ReferencedLibraryTests : LibraryTests; #file TestLibrary/TestLibrary.csproj false $TargetFramework$ #file TestLibrary/LibraryTests.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace TestLibrary; [TestClass] public abstract class LibraryTests { [TestMethod] public void TestFromReferencedLibrary() { Assert.AreEqual("TestLibrary", typeof(LibraryTests).Assembly.GetName().Name); } } """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( "MSTestSdkTestApplicationWithReferencedTestLibrary", TestApplicationWithReferencedTestLibrary .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent)); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom( testAsset.TargetAssetPath, "MSTestSdkTestApplication", TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); string libraryAssets = await File.ReadAllTextAsync( Path.Combine(testAsset.TargetAssetPath, "TestLibrary", "obj", "project.assets.json"), TestContext.CancellationToken); Assert.DoesNotContain("\"MSTest.TestAdapter/", libraryAssets); Assert.DoesNotContain("\"Microsoft.NET.Test.Sdk/", libraryAssets); Assert.DoesNotContain("\"Microsoft.Testing.Platform/", libraryAssets); Assert.DoesNotContain("\"Microsoft.Testing.Platform.MSBuild/", libraryAssets); Assert.DoesNotContain("\"Microsoft.Testing.Extensions.", libraryAssets); } [TestMethod] public async Task MSTestParallelizeScope_ClassLevel_EmitsParallelizeAttribute() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "ClassLevel")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); testHostResult.AssertOutputMatchesRegex(@"Test Parallelization enabled .* \(Workers: \d+, Scope: ClassLevel\)"); } [TestMethod] public async Task MSTestParallelizeScope_MethodLevelWithWorkers_EmitsParallelizeAttribute() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ MethodLevel 3 """)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); testHostResult.AssertOutputContains("Test Parallelization enabled"); testHostResult.AssertOutputContains("(Workers: 3, Scope: MethodLevel)"); } [TestMethod] public async Task MSTestParallelizeWorkers_Only_EmitsParallelizeAttributeWithDefaultScope() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "2")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); // ParallelizeAttribute defaults Scope to ClassLevel when only Workers is provided. testHostResult.AssertOutputContains("Test Parallelization enabled"); testHostResult.AssertOutputContains("(Workers: 2, Scope: ClassLevel)"); } [TestMethod] public async Task MSTestParallelizeWorkers_Zero_IsAcceptedAndEmitsParallelizeAttribute() { // Workers=0 is a boundary: the regex ^\d+$ accepts it and ParallelizeAttribute // interprets it as "use the number of available processors". Make sure the build // succeeds and the runtime picks it up. using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "0")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); testHostResult.AssertOutputContains("Test Parallelization enabled"); // 0 workers means "use Environment.ProcessorCount" so the actual reported value is the host CPU count, not 0. testHostResult.AssertOutputMatchesRegex(@"\(Workers: \d+, Scope: ClassLevel\)"); } [TestMethod] public async Task MSTestParallelizeScope_None_EmitsDoNotParallelizeAttribute() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "None")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); // DoNotParallelize disables parallelization entirely. testHostResult.AssertOutputDoesNotContain("Test Parallelization enabled"); } [TestMethod] public async Task MSTestParallelizeScope_GenerateAssemblyInfoFalse_EmitsWarning() { // Without GenerateAssemblyInfo, the AssemblyAttribute items are not emitted, so the // properties would silently do nothing. Verify that the build succeeds but a warning // is reported to make this case discoverable. using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ false ClassLevel """)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", warnAsError: false, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); compilationResult.AssertOutputContains("MSTestParallelizeScope and MSTestParallelizeWorkers require GenerateAssemblyInfo to be true to take effect."); } [TestMethod] public async Task MSTestParallelizeScope_InvalidValue_FailsBuild() { // An invalid scope value is emitted into the generated assembly attribute and rejected by the C# compiler (CS0117). using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "NotAValidValue")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"build {testAsset.TargetAssetPath}", failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(1); compilationResult.AssertOutputContains("CS0117"); compilationResult.AssertOutputContains("NotAValidValue"); } [TestMethod] public async Task MSTestParallelizeWorkers_NonInteger_FailsBuild() { // A non-integer Workers value is emitted into the generated assembly attribute and rejected by the C# compiler (CS0103). using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "abc")); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"build {testAsset.TargetAssetPath}", failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(1); compilationResult.AssertOutputContains("CS0103"); compilationResult.AssertOutputContains("abc"); } [TestMethod] public async Task MSTestParallelizeScope_None_With_Workers_FailsBuild() { using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCode .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ None 2 """)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync( $"build {testAsset.TargetAssetPath}", failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(1); compilationResult.AssertOutputContains("Property MSTestParallelizeWorkers cannot be set when MSTestParallelizeScope is 'None'."); } [TestMethod] public async Task MSTestParallelizeScope_VSTestRunner_BuildSucceeds() { // The targets emit assembly attributes via WriteCodeFragment, which is runner-agnostic. // Make sure the targets don't interfere with VSTest projects (build only — observing the // "Test Parallelization enabled" diagnostic is MTP-specific). using TestAsset testAsset = await TestAsset.GenerateAssetAsync( AssetName, SingleTestSourceCodeVSTest .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", """ true MethodLevel 2 """)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); } // Verifies that the MSTestParallelizeScope/MSTestParallelizeWorkers properties also work for // projects that don't use MSTest.Sdk (i.e. they reference MSTest.TestAdapter directly via the // MSTest meta-package), since the targets logic ships in MSTest.TestAdapter. [TestMethod] public async Task MSTestParallelizeScope_NonSdkProject_EmitsParallelizeAttribute() { const string NonSdkSource = """ #file MSTestPlain.csproj $TargetFramework$ Exe true x64 $(NoWarn);NU1507 MethodLevel 3 #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MSTestPlainTest { [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod1() { } } } """; const string PlainAssetName = "MSTestPlain"; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( PlainAssetName, NonSdkSource .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, PlainAssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); testHostResult.AssertOutputContains("Test Parallelization enabled"); testHostResult.AssertOutputContains("(Workers: 3, Scope: MethodLevel)"); } // Verifies that MSTest.Sdk can be layered on top of a different base SDK (here Microsoft.NET.Sdk.Web) // using manual SDK imports without emitting MSB4011 duplicate-import warnings. See https://github.com/microsoft/testfx/issues/9562. [TestMethod] public async Task MSTestSdk_LayeredOnTopOfWebSdk_BuildsWithoutDuplicateImportWarnings() { const string MixedSdkSource = """ #file MSTestWeb.csproj true $TargetFramework$ x64 $(NoWarn);NU1507 #file global.json { "msbuild-sdks": { "MSTest.Sdk": "$MSTestVersion$" } } #file UnitTest1.cs using Microsoft.AspNetCore.Builder; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MSTestWebTest { [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod1() { WebApplicationBuilder builder = WebApplication.CreateBuilder(); Assert.IsNotNull(builder); } } } """; const string MixedAssetName = "MSTestWeb"; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( MixedAssetName, MixedSdkSource .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent)); DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath}", cancellationToken: TestContext.CancellationToken); compilationResult.AssertExitCodeIs(0); compilationResult.AssertOutputDoesNotContain("MSB4011"); var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, MixedAssetName, TargetFrameworks.NetCurrent, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP is Windows-only.")] public async Task MSTestSdk_ModernUwp_DefaultsToMtpWithAppModelController() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "ModernUwpSdk", """ true <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=true"); result.AssertOutputContains("Controller=mstest-appmodel-controller.exe"); result.AssertOutputContains("ControllerTfm=net9.0"); result.AssertOutputContains("ControllerExtensions=msbuild;packagedapp;codecoverage;trx;mstest;EndContract"); result.AssertOutputContains("MSTest.TestAdapter"); result.AssertOutputContains("MSTest.TestFramework"); result.AssertOutputContains("Microsoft.Testing.Extensions.PackagedApp"); result.AssertOutputContains("TestContainer"); result.AssertOutputDoesNotContain("Microsoft.NET.Test.Sdk"); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP is Windows-only.")] public async Task MSTestSdk_ModernUwp_AllowsExplicitMtpSelection() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "ModernUwpMtpSdk", """ true true false <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=true"); result.AssertOutputContains("Controller=mstest-appmodel-controller.exe"); result.AssertOutputContains("Microsoft.Testing.Extensions.PackagedApp"); result.AssertOutputDoesNotContain("Microsoft.NET.Test.Sdk"); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP XAML references are supported only on Windows.")] public async Task MSTestSdk_UwpXamlReferencesWithoutUwpTools_UsesDirectMtpRunner() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UwpXamlReferencesSdk", """ true false """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=true;GenerateHelper=true;PackagedApp=false;OutputType=Exe"); result.AssertOutputContains(";Controller=;ControllerTfm=;ControllerExtensions="); result.AssertOutputContains("Capabilities=TestingPlatformServer"); result.AssertOutputContains("MSTest.TestAdapter"); result.AssertOutputDoesNotContain("Microsoft.Testing.Extensions.PackagedApp"); result.AssertOutputDoesNotContain("Microsoft.NET.Test.Sdk"); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP XAML references are supported only on Windows.")] public async Task MSTestSdk_UwpXamlReferencesWithoutUwpTools_BuildsAndRunsDirectMtpRunner() { const string DirectMtpAssetName = "UwpXamlReferencesDirectMtp"; const string Source = """ #file $AssetName$.csproj $TargetFramework$ x64 true false true false $(NoWarn);NU1507 #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace UwpXamlReferencesDirectMtp; [TestClass] public sealed class UnitTest1 { [TestMethod] public void UseUwpToolsFalse_RunsThroughDirectMtp() => Assert.AreEqual(4, 2 + 2); } """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( DirectMtpAssetName, Source .PatchCodeWithReplace("$AssetName$", DirectMtpAssetName) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", $"{TargetFrameworks.NetCurrent}-windows10.0.26100.0")); string binlogPath = Path.Combine(testAsset.TargetAssetPath, $"{DirectMtpAssetName}.binlog"); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build -c {BuildConfiguration.Release} {testAsset.TargetAssetPath} -bl:\"{binlogPath}\"", workingDirectory: testAsset.TargetAssetPath, cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string resolvedAssetsReport = Path.Combine(testAsset.TargetAssetPath, "resolved-mstest-assets.txt"); Assert.IsTrue( File.Exists(resolvedAssetsReport), $"The resolved MSTest asset report '{resolvedAssetsReport}' was not created. Binlog: '{binlogPath}'."); string[] resolvedAssets = await File.ReadAllLinesAsync(resolvedAssetsReport, TestContext.CancellationToken); Assert.Contains("UseUwp=true", resolvedAssets); Assert.Contains("UseUwpTools=false", resolvedAssets); Assert.Contains("PackagedApp=false", resolvedAssets); Assert.Contains("TestingPlatformExecutablePath=", resolvedAssets); string adapterPath = resolvedAssets.Single( path => path.EndsWith("MSTest.TestAdapter.dll", StringComparison.OrdinalIgnoreCase)); string frameworkExtensionsPath = resolvedAssets.Single( path => path.EndsWith("MSTest.TestFramework.Extensions.dll", StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain($"{Path.DirectorySeparatorChar}uwp{Path.DirectorySeparatorChar}", adapterPath, adapterPath); Assert.DoesNotContain($"{Path.DirectorySeparatorChar}uwp{Path.DirectorySeparatorChar}", frameworkExtensionsPath, frameworkExtensionsPath); Assert.DoesNotContain( path => path.Contains("Microsoft.Testing.Extensions.PackagedApp", StringComparison.OrdinalIgnoreCase), resolvedAssets); Assert.DoesNotContain( path => path.Contains("mstest-appmodel-controller", StringComparison.OrdinalIgnoreCase), resolvedAssets); var testHost = TestHost.LocateFrom( testAsset.TargetAssetPath, DirectMtpAssetName, $"{TargetFrameworks.NetCurrent}-windows10.0.26100.0", buildConfiguration: BuildConfiguration.Release); Assert.AreEqual($"{DirectMtpAssetName}.exe", Path.GetFileName(testHost.FullName), ignoreCase: true); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(0); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP XAML references are supported only on Windows.")] public async Task MSTestSdk_UwpXamlReferencesWithoutUwpTools_WithApplicationDefinition_UsesSelfHostedDirectMtp() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UwpXamlReferencesSelfHostedSdk", """ true false <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=false;OutputType=Exe"); result.AssertOutputContains(";Controller=;ControllerTfm=;ControllerExtensions="); result.AssertOutputDoesNotContain("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] [TestCategory("WindowsApplicationModel")] [OSCondition(OperatingSystems.Windows, IgnoreMessage = "UWP XAML references are supported only on Windows.")] public async Task MSTestSdk_UnpackagedWinUIWithUwpXamlReferences_DoesNotConflictWithUwpTools() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UnpackagedWinUIUwpXamlSdk", """ true None true false <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=false;OutputType=Exe"); result.AssertOutputContains(";Controller=;ControllerTfm=;ControllerExtensions="); result.AssertOutputDoesNotContain("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] public async Task MSTestSdk_UnpackagedWinUI_DefaultsToMtpWithoutPackagedAppLauncher() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UnpackagedWinUISdk", """ true None <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=false"); result.AssertOutputContains("OutputType=Exe"); result.AssertOutputContains("Capabilities=TestingPlatformServer"); result.AssertOutputContains("TestContainer"); result.AssertOutputDoesNotContain("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] public async Task MSTestSdk_UnpackagedWinUIWithExplicitPackagedAppExtension_KeepsDirectLaunch() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UnpackagedWinUIWithPackagedAppSdk", """ true None true <_IncludeApplicationDefinition>true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=true"); result.AssertOutputContains(";Controller=;ControllerTfm=;ControllerExtensions="); result.AssertOutputContains("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] public async Task MSTestSdk_ConsoleWithExplicitPackagedAppExtension_KeepsDirectLaunch() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "ConsoleWithPackagedAppSdk", """ true """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=true;GenerateHelper=true;PackagedApp=true"); result.AssertOutputContains(";Controller=;ControllerTfm=;ControllerExtensions="); result.AssertOutputContains("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] public async Task MSTestSdk_UnpackagedWinUI_RejectsVSTest() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "UnpackagedWinUIVSTestSdk", """ true None true """, failIfReturnValueIsNotZero: false, target: "Build"); Assert.AreNotEqual(0, result.ExitCode); result.AssertOutputContains("VSTest does not support unpackaged WinUI test applications."); } [TestMethod] [DataRow("Windows_NT", "mstest-appmodel-controller.exe", "net9.0", "msbuild;packagedapp;codecoverage;trx;mstest")] [DataRow("Unix", "", "", "")] public async Task MSTestSdk_PackagedWinUI_SelectsLauncherOnlyOnWindows( string operatingSystem, string controller, string controllerTfm, string controllerExtensions) { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "PackagedWinUISdk", $$""" true <_IncludeApplicationDefinition>true {{operatingSystem}} """); result.AssertOutputContains( "WindowsTestContract:UseVSTest=false;GenerateEntryPoint=false;GenerateHelper=true;PackagedApp=true;OutputType=Exe;"); result.AssertOutputContains( $";Controller={controller};ControllerTfm={controllerTfm};ControllerExtensions={controllerExtensions};EndContract"); result.AssertOutputContains("Microsoft.Testing.Extensions.PackagedApp"); } [TestMethod] public async Task MSTestSdk_WinUISeparateHost_KeepsGeneratedEntryPoint() { DotnetMuxerResult result = await EvaluateWindowsApplicationModelAsync( "SeparateHostWinUISdk", """ true None """); result.AssertOutputContains("WindowsTestContract:UseVSTest=false;GenerateEntryPoint=true;GenerateHelper=true;PackagedApp=false"); result.AssertOutputContains("OutputType=Exe"); } private static void AssertSatelliteCultures(string outputDirectory, string resourceAssemblyName, string[] expectedCultures) { string[] actualCultures = Directory.GetFiles(outputDirectory, resourceAssemblyName, SearchOption.AllDirectories) .Select(path => Path.GetRelativePath(outputDirectory, Path.GetDirectoryName(path)!)) .Order(StringComparer.OrdinalIgnoreCase) .ToArray(); string[] expected = expectedCultures.Order(StringComparer.OrdinalIgnoreCase).ToArray(); Assert.AreSequenceEqual( expected, actualCultures, $"Unexpected cultures for {resourceAssemblyName}. Expected: '{string.Join(";", expected)}'. Actual: '{string.Join(";", actualCultures)}'."); } private async Task EvaluateWindowsApplicationModelAsync( string assetName, string applicationModelProperties, bool failIfReturnValueIsNotZero = true, string target = "PrintWindowsTestContract") { const string Source = """ #file $AssetName$.csproj $TargetFramework$ Exe true $(NoWarn);NETSDK1201;NU1507 $ApplicationModelProperties$ """; using TestAsset testAsset = await TestAsset.GenerateAssetAsync( assetName, Source .PatchCodeWithReplace("$AssetName$", assetName) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ApplicationModelProperties$", applicationModelProperties)); string binlogPath = Path.Combine(testAsset.TargetAssetPath, $"{assetName}.binlog"); return await DotnetCli.RunAsync( $"build {testAsset.TargetAssetPath} -restore -t:{target} -bl:\"{binlogPath}\"", failIfReturnValueIsNotZero: failIfReturnValueIsNotZero, cancellationToken: TestContext.CancellationToken); } }