// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Deployment; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestTools.UnitTesting; using ExecutionScope = Microsoft.VisualStudio.TestTools.UnitTesting.ExecutionScope; namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; internal partial class TestExecutionManager { /// /// Execute the parameter tests. /// /// Tests to execute. /// Host-provided test-run directory and run settings XML. /// Logger used to report test messages back to the host. /// Recorder used to report test results back to the host. /// Provider for the test filter, or for no filter. /// Indicates if deployment is done. internal virtual async Task ExecuteTestsAsync(IEnumerable tests, DeploymentContext deploymentContext, IAdapterMessageLogger messageLogger, ITestResultRecorder testResultRecorder, ITestElementFilterProvider? filterProvider, bool isDeploymentDone) { _testResultRecorder = testResultRecorder; _testElementFilterProvider = filterProvider; InitializeRandomTestOrder(messageLogger); var testsBySource = (from test in tests group test by test.TestMethod.AssemblyName into testGroup select new { Source = testGroup.Key, Tests = (IEnumerable)testGroup }).ToArray(); if (_testOrderRandom is { } random) { Shuffle(random, testsBySource); } // Configured declarations are applied per source below, but their diagnostics have to be judged // against the whole run: a declaration naming a test in assembly A is legitimately absent from // assembly B, so warning per source would report every valid declaration as unmatched once for each // other assembly. Reported once here, against every test in the run. if (MSTestSettings.CurrentSettings.DeclaredDependencies is { Length: > 0 } declaredDependencies) { TestDependencyDeclaration.ReportUnmatchedDeclarations(declaredDependencies, tests, messageLogger); } foreach (var group in testsBySource) { _testRunCancellationToken?.ThrowIfCancellationRequested(); await ExecuteTestsInSourceAsync(group.Tests, deploymentContext, messageLogger, group.Source, isDeploymentDone).ConfigureAwait(false); } } /// /// Execute the parameter tests present in parameter source. /// /// Tests to execute. /// Host-provided test-run directory and run settings XML. /// Logger used to report test messages back to the host. /// The test container for the tests. /// Indicates if deployment is done. private async Task ExecuteTestsInSourceAsync(IEnumerable tests, DeploymentContext deploymentContext, IAdapterMessageLogger messageLogger, string source, bool isDeploymentDone) { DebugEx.Assert(!StringEx.IsNullOrEmpty(source), "Source cannot be empty"); #if !WINDOWS_UWP && !WIN_UI if (isDeploymentDone) { source = Path.Combine(PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory()!, Path.GetFileName(source)); } #endif IAdapterMessageLogger adapterMessageLogger = messageLogger; using ITestSourceHost isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, deploymentContext.RunSettingsXml); bool usesAppDomains = isolationHost is TestSourceHost { UsesAppDomain: true }; if (usesAppDomains && TestClassInstanceFactoryProvider.Current is not null) { throw new InvalidOperationException(Resource.HostedTestClassInjectionRequiresNoAppDomain); } if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsInfoEnabled) { PlatformServiceProvider.Instance.AdapterTraceLogger.Info("Created unit-test runner {0}", source); } // Default test set is filtered tests based on user provided filter criteria bool filterHasError = false; ITestElementFilter? filter = _testElementFilterProvider?.GetTestElementFilter(adapterMessageLogger, out filterHasError); if (filterHasError) { // Bail out without processing everything else below. return; } // this is done so that appropriate values of test context properties are set at source level // and are merged with session level parameters IDictionary sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source); if (_sessionParameters is { Count: > 0 }) { sourceLevelParameters = _sessionParameters.ConcatWithOverwrites(sourceLevelParameters); } _testRunCancellationToken?.ThrowIfCancellationRequested(); TestAssemblySettingsProvider? sourceSettingsProvider = null; try { sourceSettingsProvider = isolationHost.CreateInstanceForType( typeof(TestAssemblySettingsProvider), null) as TestAssemblySettingsProvider; } catch (Exception ex) { if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsInfoEnabled) { PlatformServiceProvider.Instance.AdapterTraceLogger.Info("Could not create TestAssemblySettingsProvider instance in child app-domain", ex); } } TestAssemblySettings sourceSettings = (sourceSettingsProvider != null) ? sourceSettingsProvider.GetSettings(source) : new TestAssemblySettings(); int parallelWorkers = sourceSettings.Workers; ExecutionScope parallelScope = sourceSettings.Scope; UnitTestElement[] testsToRun = [.. tests.Where(t => MatchTestFilter(filter, t, source))]; if (_testOrderRandom is { } sourceRandom) { Shuffle(sourceRandom, testsToRun); } UnitTestElement[] unitTestElements = [.. testsToRun.Select(e => e.WithUpdatedSource(source))]; // Create an instance of a type defined in adapter so that adapter gets loaded in the child app domain var testRunner = (UnitTestRunner)isolationHost.CreateInstanceForType( typeof(UnitTestRunner), [MSTestSettings.CurrentSettings, unitTestElements])!; // Ensures that the cancellation token gets through AppDomain boundary. _testRunCancellationToken?.Register(static state => ((UnitTestRunner)state!).Cancel(), testRunner); if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value; } if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value; } bool parallelizationEnabled = !MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0; // Sorting the graph's input by name is what makes name order the *tie-breaker* within the dependency // order, rather than something the graph overrides: OrderTopologically breaks ties by position in this // array, so tests that are simultaneously ready still run in name order. Doing it here rather than in // the runner is deliberate - re-sorting a chunk after the topological pass would undo the ordering the // graph just established. Ordering keys mirror the historical VSTest ManagedType/ManagedMethod // test-case properties, which are only populated when the test method carries managed method metadata. if (MSTestSettings.CurrentSettings.OrderTestsByNameInClass && !MSTestSettings.CurrentSettings.RandomizeTestOrder) { testsToRun = [ .. testsToRun .OrderBy(t => t.TestMethod.HasManagedMethodAndTypeProperties ? t.TestMethod.ManagedTypeName : null, StringComparer.Ordinal) .ThenBy(t => t.TestMethod.HasManagedMethodAndTypeProperties ? t.TestMethod.ManagedMethodName : null, StringComparer.Ordinal), ]; } // Edges declared in testconfig.json are merged onto the elements first, so that from the graph's // point of view a configured dependency is indistinguishable from an attribute-declared one. if (MSTestSettings.CurrentSettings.DeclaredDependencies is { Length: > 0 } declaredDependencies) { TestDependencyDeclaration.ApplyAll(declaredDependencies, testsToRun, adapterMessageLogger: null); } // Returns null - and so leaves every run that does not use [DependsOn] on exactly the path it uses // today - unless at least one test in this source declares a dependency. var dependencyGraph = TestDependencyGraph.Build(testsToRun, parallelScope, parallelizationEnabled); if (parallelizationEnabled) { // Parallelization is enabled. Let's do further classification for sets. adapterMessageLogger.SendMessage( MessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, source, parallelWorkers, parallelScope)); if (dependencyGraph is not null) { await ExecuteTestsWithDependencyGraphAsync(dependencyGraph, adapterMessageLogger, source, sourceLevelParameters, testRunner, usesAppDomains, parallelWorkers).ConfigureAwait(false); } else { await ExecuteParallelizedTestsAsync(testsToRun, adapterMessageLogger, source, sourceLevelParameters, testRunner, usesAppDomains, parallelWorkers, parallelScope).ConfigureAwait(false); } } else if (dependencyGraph is not null) { await ExecuteTestsWithDependencyGraphAsync(dependencyGraph, adapterMessageLogger, source, sourceLevelParameters, testRunner, usesAppDomains, parallelWorkers: 1).ConfigureAwait(false); } else { await ExecuteTestsWithTestRunnerAsync(testsToRun, adapterMessageLogger, source, sourceLevelParameters, testRunner, usesAppDomains).ConfigureAwait(false); } if (PlatformServiceProvider.Instance.IsGracefulStopRequested) { testRunner.ForceCleanup(sourceLevelParameters!, new RemotingMessageLogger(adapterMessageLogger)); } if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsInfoEnabled) { PlatformServiceProvider.Instance.AdapterTraceLogger.Info("Executed tests belonging to source {0}", source); } } }