// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.IO.Compression; using System.Xml.Linq; using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; namespace MSTest.Acceptance.IntegrationTests; /// /// MSTest.SourceGeneration is most commonly used in Native AOT and trimming scenarios /// (see and ), but the package itself /// is a plain Roslyn source generator that should work in any SDK-style project. These /// acceptance tests pin that behavior: referencing the package on a non-AOT, non-trimmed /// project must still build, emit the generated metadata file, and successfully run tests /// through the source-generated ReflectionMetadataHook module initializer. /// [TestClass] public class SourceGenerationNonAotTests : AcceptanceTestBase { private const string AssetName = "MSTestSourceGenNonAot"; private const string SdkAssetName = "MSTestSourceGenSdkNonAot"; // Source code for a non-AOT, non-trimmed test project that references MSTest.SourceGeneration. // EmitCompilerGeneratedFiles is enabled so we can statically assert the generator ran and // wrote out the expected metadata file alongside the assembly. private const string SourceCode = """ #file MSTestSourceGenNonAot.csproj $TargetFramework$ enable enable Exe preview true ReflectionFree true #file TestClass1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MyTests; [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod1() { } [TestMethod] [DataRow(0, 1)] public void TestMethod2(int a, int b) { } // Exercises the source-generated DynamicData accessor (property source) at runtime: if the generated // DynamicDataSourceResolver registration produced the wrong data, this test would fail or not run. public static IEnumerable Data => new[] { new object[] { 1, 2 }, new object[] { 3, 4 } }; [TestMethod] [DynamicData(nameof(Data))] public void TestMethod3(int a, int b) { Assert.AreEqual(a + 1, b); } [TestMethod] [DataRow(1, "managed", true)] public async Task TestMethod4(int size, string category, bool enabled) { await Task.Yield(); Assert.AreEqual(1, size); Assert.AreEqual("managed", category); Assert.IsTrue(enabled); } [TestMethod] public void Overload() { } [TestMethod] [DataRow(42)] public void Overload(int value) { Assert.AreEqual(42, value); } [TestMethod] [DataRow(null)] public void NullDataRow(string? value) { Assert.IsNull(value); } } """; private const string CentralPackageManagementSourceCode = """ #file Directory.Packages.props true false """; private const string SdkSourceCode = """ #file MSTestSourceGenSdkNonAot.csproj $TargetFramework$ $IsTestApplication$ $PublishAot$ $UseVSTest$ $EnableMicrosoftTestingPlatformProperty$ $EnableMSTestSourceGenerationProperty$ true $ExtraItems$ #file TestClass1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MyTests; [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod1() { } } """; private const string TestingPlatformPackageReference = """ """; [TestMethod] [DynamicData(nameof(TargetFrameworks.NetForDynamicData), typeof(TargetFrameworks))] public async Task SourceGenerationNonAot_BuildsAndRunsTests_WithExitCodeZero(string tfm) { using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{AssetName}_{tfm}", SourceCode .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$MSTestSourceGenerationVersion$", MSTestSourceGenerationVersion), addPublicFeeds: true); // Plain `dotnet build` — no -r RID, no publish, no PublishAot, no PublishTrimmed. // This is the scenario the follow-up review surfaced: the source generator must also // work for users who opt in to source-generated discovery without trimming/AOT. DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release} -f {tfm}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); // Static evidence the source generator actually ran in the build (not just that // the package was restored). EmitCompilerGeneratedFiles writes the generator // output under obj///generated///. // This asset explicitly selects ReflectionFree mode because the assertions below verify // its compact registry and registration shape. string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.IsNotEmpty(generatedFiles, $"the source generator should have emitted a '*MSTestReflectionMetadata*.g.cs' file under '{objGenerated}'"); string registry = File.ReadAllText(generatedFiles.Single(path => path.EndsWith("MSTestReflectionMetadata.Registry.g.cs", StringComparison.Ordinal))); Assert.DoesNotContain("DataRows", registry, "DataRowAttribute instances are authoritative; a second argument-array descriptor is redundant."); Assert.DoesNotContain("ParameterNames", registry, "runtime registration only resolves overloads by parameter type."); Assert.Contains("SupportsGeneratedDescriptors = true", registry); Assert.Contains("IsDescriptorSupported = true", registry); Assert.Contains("AreGeneratedDescriptorsComplete = false", registry); string registration = File.ReadAllText(generatedFiles.Single(path => path.EndsWith("MSTestReflectionMetadata.Registration.g.cs", StringComparison.Ordinal))); Assert.Contains("availableMethods ??= type.GetMethods(memberFlags)", registration); Assert.Contains("ResolveMethod(availableMethods, method.DeclaringType, method.Name, method.ParameterTypes)", registration); Assert.Contains("methodInfo.GetCustomAttributes(typeof(AsyncStateMachineAttribute), inherit: false)", registration); Assert.Contains("methodInfo.GetCustomAttributes(typeof(DebuggerStepThroughAttribute), inherit: false)", registration); Assert.Contains("descriptorTestMethods[type] = descriptorMethodRoots.ToArray()", registration); Assert.Contains("descriptorTestMethods, descriptorCompleteTypes.ToArray()", registration); // Behavioral evidence: tests still discover and run when the source-generated // ReflectionMetadataHook is the only metadata provider wired in at module init. // If the hook crashed during ModuleInitializer or swapped in a broken provider, the // test host would fail before printing a summary. (We assert the full count to also // catch silent discovery regressions where tests are not picked up.) var testHost = TestHost.LocateFrom(generator.TargetAssetPath, AssetName, tfm, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 8, skipped: 0); testHostResult.AssertExitCodeIs(0); TestHostResult nullRowResult = await testHost.ExecuteAsync( "--filter FullyQualifiedName~NullDataRow", cancellationToken: TestContext.CancellationToken); nullRowResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); nullRowResult.AssertExitCodeIs(0); } [TestMethod] [DataRow(false)] [DataRow(true)] public async Task MSTestSdk_SourceGenerationOptIn_LibraryBuilds(bool useCentralPackageManagement) { string tfm = TargetFrameworks.NetCurrent; using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_Library_{useCentralPackageManagement}", (useCentralPackageManagement ? SdkSourceCode + CentralPackageManagementSourceCode : SdkSourceCode) .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$IsTestApplication$", "false") .PatchCodeWithReplace("$PublishAot$", "false") .PatchCodeWithReplace("$UseVSTest$", "false") .PatchCodeWithReplace("$EnableMicrosoftTestingPlatformProperty$", string.Empty) .PatchCodeWithReplace( "$EnableMSTestSourceGenerationProperty$", "true") .PatchCodeWithReplace("$ExtraItems$", string.Empty) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release} -f {tfm}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.HasCount(3, generatedFiles, $"the MSTest.Sdk source-generation opt-in should have emitted exactly three metadata files under '{objGenerated}'"); } [TestMethod] [DataRow(false)] [DataRow(true)] public async Task MSTestSdk_SourceGenerationOptIn_ApplicationBuildsAndRuns(bool useCentralPackageManagement) { string tfm = TargetFrameworks.NetCurrent; using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_Application_{useCentralPackageManagement}", (useCentralPackageManagement ? SdkSourceCode + CentralPackageManagementSourceCode : SdkSourceCode) .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$IsTestApplication$", "true") .PatchCodeWithReplace("$PublishAot$", "false") .PatchCodeWithReplace("$UseVSTest$", "false") .PatchCodeWithReplace( "$EnableMicrosoftTestingPlatformProperty$", "true") .PatchCodeWithReplace( "$EnableMSTestSourceGenerationProperty$", "true") .PatchCodeWithReplace("$ExtraItems$", string.Empty) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release} -f {tfm}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.HasCount(3, generatedFiles, $"the MSTest.Sdk source-generation opt-in should have emitted exactly three metadata files under '{objGenerated}'"); var testHost = TestHost.LocateFrom(generator.TargetAssetPath, SdkAssetName, tfm, buildConfiguration: BuildConfiguration.Release); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); testHostResult.AssertExitCodeIs(0); } [TestMethod] public async Task MSTestSdk_SourceGenerationOptIn_NativeAotLibraryBuilds() { string tfm = TargetFrameworks.NetCurrent; using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_NativeAotLibrary", SdkSourceCode .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$IsTestApplication$", "false") .PatchCodeWithReplace("$PublishAot$", "true") .PatchCodeWithReplace("$UseVSTest$", "false") .PatchCodeWithReplace("$EnableMicrosoftTestingPlatformProperty$", string.Empty) .PatchCodeWithReplace("$EnableMSTestSourceGenerationProperty$", string.Empty) .PatchCodeWithReplace("$ExtraItems$", TestingPlatformPackageReference) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.HasCount(3, generatedFiles, $"the NativeAOT library source generator should have emitted exactly three metadata files under '{objGenerated}'"); } [TestMethod] public async Task MSTestSdk_SourceGenerationOptIn_VSTestTakesPrecedenceOverNativeAot() { string tfm = TargetFrameworks.NetCurrent; using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_VSTestNativeAot", SdkSourceCode .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$IsTestApplication$", "true") .PatchCodeWithReplace("$PublishAot$", "true") .PatchCodeWithReplace("$UseVSTest$", "true") .PatchCodeWithReplace("$EnableMicrosoftTestingPlatformProperty$", string.Empty) .PatchCodeWithReplace( "$EnableMSTestSourceGenerationProperty$", "true") .PatchCodeWithReplace("$ExtraItems$", string.Empty) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.HasCount(3, generatedFiles, $"the VSTest source-generation opt-in should have emitted exactly three metadata files under '{objGenerated}'"); } [TestMethod] public async Task MSTestSdk_SourceGenerationOptIn_DoesNotFlowTransitively() { string tfm = TargetFrameworks.NetCurrent; using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_Pack", SdkSourceCode .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$IsTestApplication$", "false") .PatchCodeWithReplace("$PublishAot$", "false") .PatchCodeWithReplace("$UseVSTest$", "false") .PatchCodeWithReplace("$EnableMicrosoftTestingPlatformProperty$", string.Empty) .PatchCodeWithReplace( "$EnableMSTestSourceGenerationProperty$", "true") .PatchCodeWithReplace("$ExtraItems$", string.Empty) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); string packageDirectory = Path.Combine(generator.TargetAssetPath, "packages"); DotnetMuxerResult packResult = await DotnetCli.RunAsync( $"pack {generator.TargetAssetPath} -c {BuildConfiguration.Release} -p:IsPackable=true -p:PackageVersion=1.0.0-test -o \"{packageDirectory}\"", cancellationToken: TestContext.CancellationToken); packResult.AssertExitCodeIs(0); string package = Directory.GetFiles(packageDirectory, "*.nupkg").Single(); using ZipArchive archive = ZipFile.OpenRead(package); ZipArchiveEntry nuspecEntry = archive.Entries.Single(entry => entry.FullName.EndsWith(".nuspec", StringComparison.Ordinal)); using Stream nuspecStream = nuspecEntry.Open(); var nuspec = XDocument.Load(nuspecStream); string[] dependencyIds = nuspec .Descendants() .Where(element => element.Name.LocalName == "dependency") .Select(element => element.Attribute("id")?.Value) .OfType() .ToArray(); Assert.Contains("MSTest.TestAdapter", dependencyIds); Assert.DoesNotContain("MSTest.SourceGeneration", dependencyIds); } [TestMethod] public async Task MSTestSdk_SourceGenerationOptIn_RejectsNetStandardTestLibrary() { using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{SdkAssetName}_NetStandard", SdkSourceCode .PatchCodeWithReplace("$TargetFramework$", "netstandard2.0") .PatchCodeWithReplace("$IsTestApplication$", "false") .PatchCodeWithReplace("$PublishAot$", "false") .PatchCodeWithReplace("$UseVSTest$", "false") .PatchCodeWithReplace("$EnableMicrosoftTestingPlatformProperty$", string.Empty) .PatchCodeWithReplace( "$EnableMSTestSourceGenerationProperty$", "true") .PatchCodeWithReplace("$ExtraItems$", string.Empty) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release}", failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(1); buildResult.AssertOutputContains( "MSTest source generation is not supported for .NET Standard target frameworks because the required MSTest.TestAdapter runtime hooks are unavailable."); } public TestContext TestContext { get; set; } = null!; private const string MetadataSourceCode = """ #file MSTestSourceGenNonAotMetadata.csproj $TargetFramework$ enable enable Exe preview true ReflectionFree true #file MetadataTests.cs using Microsoft.VisualStudio.TestTools.UnitTesting; namespace MyTests; [TestClass] public sealed class MethodMetadataTests { [TestMethod] [TestCategory("GeneratedMetadata")] public void CategorizedTest() { } [TestMethod] [Ignore("generated method ignore")] public void IgnoredMethod() => Assert.Fail("The generated method-level IgnoreAttribute was not honored."); } [TestCategory("InheritedTypeMetadata")] public abstract class CategorizedBase { } [TestClass] public sealed class InheritedCategorizedTests : CategorizedBase { [TestMethod] public void CategorizedThroughBaseClass() { } } """; [TestMethod] [DynamicData(nameof(TargetFrameworks.NetForDynamicData), typeof(TargetFrameworks))] public async Task SourceGenerationNonAot_HonorsGeneratedInheritedTypeAndMethodMetadata(string tfm) { using TestAsset generator = await TestAsset.GenerateAssetAsync( $"{AssetName}Metadata_{tfm}", MetadataSourceCode .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$TargetFramework$", tfm) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$MSTestSourceGenerationVersion$", MSTestSourceGenerationVersion), addPublicFeeds: true); DotnetMuxerResult buildResult = await DotnetCli.RunAsync( $"build {generator.TargetAssetPath} -c {BuildConfiguration.Release} -f {tfm}", cancellationToken: TestContext.CancellationToken); buildResult.AssertExitCodeIs(0); string objGenerated = Path.Combine(generator.TargetAssetPath, "obj", "Release", tfm, "generated"); string[] generatedFiles = Directory.Exists(objGenerated) ? Directory.GetFiles(objGenerated, "*MSTestReflectionMetadata*.g.cs", SearchOption.AllDirectories) : []; Assert.IsNotEmpty(generatedFiles, $"the reflection-free source generator should have emitted metadata under '{objGenerated}'"); var testHost = TestHost.LocateFrom(generator.TargetAssetPath, "MSTestSourceGenNonAotMetadata", tfm, buildConfiguration: BuildConfiguration.Release); TestHostResult fullRun = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); fullRun.AssertOutputContainsSummary(failed: 0, passed: 2, skipped: 1); fullRun.AssertExitCodeIs(0); TestHostResult categoryRun = await testHost.ExecuteAsync( "--filter TestCategory=GeneratedMetadata", cancellationToken: TestContext.CancellationToken); categoryRun.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); categoryRun.AssertExitCodeIs(0); TestHostResult inheritedCategoryRun = await testHost.ExecuteAsync( "--filter TestCategory=InheritedTypeMetadata", cancellationToken: TestContext.CancellationToken); inheritedCategoryRun.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); inheritedCategoryRun.AssertExitCodeIs(0); } }