// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using AwesomeAssertions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Resources; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.TestableImplementations; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Moq; using TestFramework.ForTestingMSTest; using ExecutionScope = Microsoft.VisualStudio.TestTools.UnitTesting.ExecutionScope; using FixtureKind = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers.FixtureKind; using TimeoutInfo = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution.TimeoutInfo; #pragma warning disable CS0618 // Type or member is obsolete namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests; public class MSTestSettingsTests : TestContainer { private readonly TestablePlatformServiceProvider _testablePlatformServiceProvider; private readonly Mock _mockDiscoveryContext; private readonly Mock _mockRunSettings; private readonly Mock _mockMessageLogger; public MSTestSettingsTests() { _testablePlatformServiceProvider = new TestablePlatformServiceProvider(); _mockDiscoveryContext = new Mock(); _mockRunSettings = new Mock(); _mockMessageLogger = new Mock(); PlatformServiceProvider.Instance = _testablePlatformServiceProvider; } protected override void Dispose(bool disposing) { if (!IsDisposed) { base.Dispose(disposing); PlatformServiceProvider.Instance = null; } } #region Property validation. public void MapInconclusiveToFailedIsByDefaultFalseWhenNotSpecified() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); } public void MapNotRunnableToFailedIsByDefaultTrueWhenNotSpecified() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); } public void MapInconclusiveToFailedShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ True """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); } public void RunSettings_WithInvalidValues_GettingAWarningForEachInvalidSetting() { string runSettingsXml = """ 3 timeout timeout timeout timeout timeout 3 timeout 3 3 3 3 timeout """; MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger()); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, It.IsAny()), Times.Exactly(13)); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'CooperativeCancellationTimeout', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'TestCleanupTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'ClassCleanupTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'AssemblyCleanupTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'CaptureTraceOutput', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'MapNotRunnableToFailed', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'TestInitializeTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'ClassInitializeTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'AssemblyInitializeTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'TestTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'TreatDiscoveryWarningsAsErrors', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'MapInconclusiveToFailed', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'ConsiderEmptyDataSourceAsInconclusive', setting will be ignored."), Times.Once); } public void RunSettings_WithZeroTimeout_GettingAWarningAndTimeoutIsNotSet() { // "0" parses as an integer but is rejected by the strictly-positive guard (issue #9403). string runSettingsXml = """ 0 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(0); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '0' for runsettings entry 'TestTimeout'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); } public void MapNotRunnableToFailedShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ True """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); } public void CaptureDebugTracesShouldBeTrueByDefault() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.CaptureDebugTraces.Should().BeTrue(); } public void CaptureAssertionFailureDiagnosticsShouldBeFalseByDefault() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.CaptureAssertionFailureDiagnostics.Should().BeFalse(); } public void CaptureAssertionFailureDiagnosticsShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ true """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.CaptureAssertionFailureDiagnostics.Should().BeTrue(); } public void CaptureDebugTracesShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ False """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.CaptureDebugTraces.Should().BeFalse(); } public void OutputCaptureModeShouldDefaultToResult() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Result); adapterSettings.CaptureDebugTraces.Should().BeTrue(); } public void OutputCaptureModeShouldMapLegacyBooleanTrueToResult() { string runSettingsXml = """ True """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Result); adapterSettings.CaptureDebugTraces.Should().BeTrue(); } public void OutputCaptureModeShouldMapLegacyBooleanFalseToNone() { string runSettingsXml = """ False """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.None); adapterSettings.CaptureDebugTraces.Should().BeFalse(); } public void OutputCaptureModeShouldBeConsumedFromRunSettingsWhenSpecifiedAsEnum() { string runSettingsXml = """ Live """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Live); adapterSettings.CaptureDebugTraces.Should().BeTrue(); } public void OutputCaptureModeShouldRejectNumericValueAndKeepDefault() { string runSettingsXml = """ 2 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; // Numeric input must not be treated as an enum ordinal (2 -> Live); it is rejected and the default is kept. adapterSettings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Result); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '2' for runsettings entry 'CaptureTraceOutput', setting will be ignored."), Times.Once); } public void TestTimeoutShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ 4000 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(4000); } public void TestTimeoutShouldBeSetToZeroIfNotSpecifiedInRunSettings() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(0); } public void TreatDiscoveryWarningsAsErrorsShouldBeTrueByDefault() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); } public void TreatDiscoveryWarningsAsErrorsShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ True """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); } public void ParallelizationSettingsShouldNotBeSetByDefault() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.HasValue.Should().BeFalse(); adapterSettings.ParallelizationScope.HasValue.Should().BeFalse(); } public void RandomizeTestOrderShouldBeFalseByDefault() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.RandomizeTestOrder.Should().BeFalse(); adapterSettings.RandomTestOrderSeed.Should().BeNull(); } public void RandomizeTestOrderShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ True 42 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.RandomizeTestOrder.Should().BeTrue(); adapterSettings.RandomTestOrderSeed.Should().Be(42); } public void RandomTestOrderSeedShouldAcceptNegativeValues() { string runSettingsXml = """ -7 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.RandomTestOrderSeed.Should().Be(-7); } public void RandomizeTestOrderShouldWarnWhenValueIsInvalid() { string runSettingsXml = """ notabool """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.RandomizeTestOrder.Should().BeFalse(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'notabool' for runsettings entry 'RandomizeTestOrder', setting will be ignored."), Times.Once); } public void RandomTestOrderSeedShouldWarnWhenValueIsInvalid() { string runSettingsXml = """ notanint """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.RandomTestOrderSeed.Should().BeNull(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'notanint' for runsettings entry 'RandomTestOrderSeed', setting will be ignored."), Times.Once); } public void PopulateSettingsShouldWarnWhenRandomizeTestOrderAndOrderTestsByNameInClassAreBothEnabled() { string runSettingsXml = """ True True """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings.CurrentSettings.RandomizeTestOrder.Should().BeTrue(); MSTestSettings.CurrentSettings.OrderTestsByNameInClass.Should().BeTrue(); _mockMessageLogger.Verify( lm => lm.SendMessage( TestMessageLevel.Warning, "Both 'RandomizeTestOrder' and 'OrderTestsByNameInClass' are set. 'OrderTestsByNameInClass' will be ignored."), Times.Once); } public void PopulateSettingsShouldNotWarnWhenOnlyRandomizeTestOrderIsEnabled() { string runSettingsXml = """ True """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings.CurrentSettings.RandomizeTestOrder.Should().BeTrue(); MSTestSettings.CurrentSettings.OrderTestsByNameInClass.Should().BeFalse(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, It.IsAny()), Times.Never); } public void GetSettingsShouldThrowIfParallelizationWorkersIsNotInt() { string runSettingsXml = """ GoneFishing """; Action act = () => MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger()); AdapterSettingsException exception = act.Should().Throw().Which; exception.Message.Should().Contain("Invalid value 'GoneFishing' specified for 'Workers'. The value should be a non-negative integer."); } public void GetSettingsShouldThrowIfParallelizationWorkersIsNegative() { string runSettingsXml = """ -1 """; Action act = () => MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger()); AdapterSettingsException exception = act.Should().Throw().Which; exception.Message.Should().Contain("Invalid value '-1' specified for 'Workers'. The value should be a non-negative integer."); } public void ParallelizationWorkersShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ 2 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.Should().Be(2); } public void ParallelizationWorkersShouldBeSetToProcessorCountWhenSetToZero() { string runSettingsXml = """ 0 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.Should().Be(Environment.ProcessorCount); } public void ParallelizationSettingsShouldBeSetToDefaultsWhenNotSet() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.Should().Be(Environment.ProcessorCount); adapterSettings.ParallelizationScope.Should().Be(ExecutionScope.ClassLevel); } public void ParallelizationSettingsShouldBeSetToDefaultsOnAnEmptyParalleizeSetting() { string runSettingsXml = """ """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.Should().Be(Environment.ProcessorCount); adapterSettings.ParallelizationScope.Should().Be(ExecutionScope.ClassLevel); } public void ParallelizationSettingsShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ 127 MethodLevel """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationWorkers.Should().Be(127); adapterSettings.ParallelizationScope.Should().Be(ExecutionScope.MethodLevel); } public void GetSettingsShouldThrowIfParallelizationScopeIsNotValid() { string runSettingsXml = """ JustParallelizeWillYou """; Action act = () => MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger()); AdapterSettingsException exception = act.Should().Throw().Which; exception.Message.Should().Contain("Invalid value 'JustParallelizeWillYou' specified for 'Scope'. Supported scopes are ClassLevel, MethodLevel."); } public void ParallelizationScopeShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ MethodLevel """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.ParallelizationScope.Should().Be(ExecutionScope.MethodLevel); } public void GetSettingsShouldThrowWhenParallelizeHasInvalidElements() { string runSettingsXml = """ Hi """; Action act = () => MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger()); AdapterSettingsException exception = act.Should().Throw().Which; exception.Message.Should().Contain("MSTestAdapter encountered an unexpected element 'Hola' in its settings 'Parallelize'. Remove this element and try again."); } public void GetSettingsShouldBeAbleToReadAfterParallelizationSettings() { string runSettingsXml = """ 12 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(12); } public void GetSettingsShouldBeAbleToReadAfterParallelizationSettingsWithData() { string runSettingsXml = """ 127 MethodLevel 12 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(12); adapterSettings.ParallelizationWorkers.Should().Be(127); adapterSettings.ParallelizationScope.Should().Be(ExecutionScope.MethodLevel); } public void GetSettingsShouldBeAbleToReadAfterParallelizationSettingsOnEmptyParallelizationNode() { string runSettingsXml = """ 12 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.TestTimeout.Should().Be(12); } public void DisableParallelizationShouldBeFalseByDefault() { string runSettingsXml = """ """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings.CurrentSettings.DisableParallelization.Should().BeFalse(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, It.IsAny()), Times.Never); } public void DisableParallelizationShouldBeConsumedFromRunSettingsWhenSpecified() { string runSettingsXml = """ True """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings.CurrentSettings.DisableParallelization.Should().BeTrue(); } public void DisableParallelization_WithInvalidValue_GettingAWarning() { string runSettingsXml = """ 3 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'DisableParallelization', setting will be ignored."), Times.Once); } #endregion #region GetSettings Tests public void GetSettingsShouldProbePlatformSpecificSettingsAlso() { string runSettingsXml = """ True """; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader actualReader) => { actualReader.Read(); actualReader.ReadInnerXml(); }); MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger()); _testablePlatformServiceProvider.MockSettingsProvider.Verify(sp => sp.Load(It.IsAny()), Times.Once); } public void GetSettingsShouldOnlyPassTheElementSubTreeToPlatformService() { string runSettingsXml = """ True True """; string expectedRunSettingXml = "True"; string? observedXml = null; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader actualReader) => { actualReader.Read(); observedXml = actualReader.ReadOuterXml(); }); MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger()); (expectedRunSettingXml == observedXml).Should().BeTrue(); } public void GetSettingsShouldBeAbleToReadSettingsAfterThePlatformServiceReadsItsSettings() { string runSettingsXml = """ True True True 12 """; bool dummyPlatformSpecificSetting = false; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader reader) => { reader.Read(); while (reader.NodeType == XmlNodeType.Element) { string elementName = reader.Name.ToUpperInvariant(); switch (elementName) { case "DUMMYPLATFORMSPECIFICSETTING": { if (bool.TryParse(reader.ReadInnerXml(), out bool result)) { dummyPlatformSpecificSetting = result; } } break; default: { reader.Skip(); break; } } } }); MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger())!; // Assert. dummyPlatformSpecificSetting.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TestTimeout.Should().Be(12); } public void GetSettingsShouldBeAbleToReadSettingsIfThePlatformServiceDoesNotUnderstandASetting() { string runSettingsXml = """ True True foobar True 12 """; bool dummyPlatformSpecificSetting = false; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader reader) => { reader.Read(); while (reader.NodeType == XmlNodeType.Element) { string elementName = reader.Name.ToUpperInvariant(); switch (elementName) { case "DUMMYPLATFORMSPECIFICSETTING": { if (bool.TryParse(reader.ReadInnerXml(), out bool result)) { dummyPlatformSpecificSetting = result; } } break; default: { reader.Skip(); break; } } } }); MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger())!; // Assert. dummyPlatformSpecificSetting.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TestTimeout.Should().Be(12); } public void GetSettingsShouldOnlyReadTheAdapterSection() { string runSettingsXml = """ True True Bad """; bool outOfScopeCall = false; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader reader) => { reader.Read(); while (reader.NodeType == XmlNodeType.Element) { string elementName = reader.Name.ToUpperInvariant(); switch (elementName) { case "BADELEMENT": { reader.ReadInnerXml(); outOfScopeCall = true; } break; default: { reader.Skip(); break; } } } }); MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger()); // Assert. outOfScopeCall.Should().BeFalse(); } public void GetSettingsShouldWorkIfThereAreCommentsInTheXML() { string runSettingsXml = """ True True True Bad """; bool dummyPlatformSpecificSetting = false; _testablePlatformServiceProvider.MockSettingsProvider.Setup(sp => sp.Load(It.IsAny())) .Callback((XmlReader reader) => { reader.Read(); while (reader.NodeType == XmlNodeType.Element) { string elementName = reader.Name.ToUpperInvariant(); switch (elementName) { case "DUMMYPLATFORMSPECIFICSETTING": { if (bool.TryParse(reader.ReadInnerXml(), out bool result)) { dummyPlatformSpecificSetting = result; } } break; default: { reader.Skip(); break; } } } }); MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger())!; // Assert. dummyPlatformSpecificSetting.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); } #endregion #region CurrentSettings tests public void CurrentSettingShouldReturnDefaultSettingsIfNotSet() { MSTestSettings.Reset(); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); // Validating the default value of a random setting. adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); } public void CurrentSettingShouldReturnCachedLoadedSettings() { string runSettingsXml = """ 12 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; MSTestSettings adapterSettings2 = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); adapterSettings.TestTimeout.Should().Be(12); (adapterSettings == adapterSettings2).Should().BeTrue(); } #endregion #region PopulateSettings tests. public void PopulateSettingsShouldFillInSettingsFromSettingsObject() { string runSettingsXml = """ False True True 12 """; var settings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName, _mockMessageLogger.Object.ToAdapterMessageLogger()); settings.Should().NotBeNull(); MSTestSettings.PopulateSettings(settings); MSTestSettings.CurrentSettings.CaptureDebugTraces.Should().BeFalse(); MSTestSettings.CurrentSettings.MapInconclusiveToFailed.Should().BeTrue(); MSTestSettings.CurrentSettings.MapNotRunnableToFailed.Should().BeTrue(); MSTestSettings.CurrentSettings.TestTimeout.Should().Be(12); } public void PopulateSettingsShouldInitializeDefaultAdapterSettingsWhenDiscoveryContextIsNull() { MSTestSettings.PopulateSettings(null, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.CaptureDebugTraces.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); } public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsIsNull() { MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.CaptureDebugTraces.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); } public void PopulateSettingsShouldInitializeDefaultSettingsWhenRunSettingsXmlIsEmpty() { _mockDiscoveryContext.Setup(md => md.RunSettings!.SettingsXml).Returns(string.Empty); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.CaptureDebugTraces.Should().BeTrue(); adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); } public void PopulateSettingsShouldInitializeSettingsToDefaultIfNotSpecified() { string runSettingsXml = """ 12 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); // Validating the default value of a random setting. adapterSettings.MapInconclusiveToFailed.Should().BeFalse(); } public void PopulateSettingsShouldInitializeSettingsFromMSTestSection() { string runSettingsXml = """ True True 12 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TestTimeout.Should().Be(12); } public void PopulateSettingsShouldInitializeSettingsFromMSTestV2Section() { string runSettingsXml = """ True True 12 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.TestTimeout.Should().Be(12); } public void PopulateSettingsShouldInitializeSettingsFromMSTestV2OverMSTestV1Section() { string runSettingsXml = """ True True False 12 """; _mockDiscoveryContext.Setup(dc => dc.RunSettings).Returns(_mockRunSettings.Object); _mockRunSettings.Setup(rs => rs.SettingsXml).Returns(runSettingsXml); MSTestSettings.PopulateSettings(_mockDiscoveryContext.Object.RunSettings?.SettingsXml, _mockMessageLogger.Object.ToAdapterMessageLogger(), null); MSTestSettings adapterSettings = MSTestSettings.CurrentSettings; adapterSettings.Should().NotBeNull(); adapterSettings.MapInconclusiveToFailed.Should().BeTrue(); adapterSettings.MapNotRunnableToFailed.Should().BeTrue(); adapterSettings.CaptureDebugTraces.Should().BeTrue(); adapterSettings.TestTimeout.Should().Be(0); } #endregion #region ConfigJson public void ConfigJson_WithInvalidValues_GettingAWarningForEachInvalidSetting() { // Arrange - setting up invalid configuration values var configDictionary = new Dictionary { { "mstest:timeout:assemblyInitialize", "timeout" }, { "mstest:timeout:assemblyCleanup", "timeout" }, { "mstest:timeout:classInitialize", "timeout" }, { "mstest:timeout:classCleanup", "timeout" }, { "mstest:timeout:testInitialize", "timeout" }, { "mstest:timeout:testCleanup", "timeout" }, { "mstest:timeout:test", "timeout" }, { "mstest:timeout:useCooperativeCancellation", "3" }, { "mstest:execution:mapInconclusiveToFailed", "3" }, { "mstest:execution:mapNotRunnableToFailed", "3" }, { "mstest:execution:treatDiscoveryWarningsAsErrors", "3" }, { "mstest:execution:considerEmptyDataSourceAsInconclusive", "3" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, It.IsAny()), Times.Exactly(12)); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'timeout:useCooperativeCancellation', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:assemblyInitialize'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:assemblyCleanup'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:classInitialize'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'execution:mapNotRunnableToFailed', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:classCleanup'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:testInitialize'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:testCleanup'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value 'timeout' for runsettings entry 'timeout:test'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'execution:treatDiscoveryWarningsAsErrors', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'execution:mapInconclusiveToFailed', setting will be ignored."), Times.Once); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '3' for runsettings entry 'execution:considerEmptyDataSourceAsInconclusive', setting will be ignored."), Times.Once); } public void ConfigJson_WithZeroTimeout_GettingAWarningAndTimeoutIsNotSet() { // Arrange - "0" parses as an integer but is rejected by the strictly-positive guard. // This is the exact scenario from issue #9403: a parseable value that silently has no effect. var configDictionary = new Dictionary { { "mstest:timeout:test", "0" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); settings.TestTimeout.Should().Be(0); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '0' for runsettings entry 'timeout:test'. The timeout must be a strictly positive integer (in milliseconds); a value of 0 or less is not allowed. Omit the entry to use the default (no timeout). The setting will be ignored."), Times.Once); } public void ConfigJson_WithValidValues_ValuesAreSetCorrectly() { // Arrange - setting up valid configuration values var configDictionary = new Dictionary { { "mstest:timeout:assemblyInitialize", "300" }, { "mstest:timeout:assemblyCleanup", "300" }, { "mstest:timeout:classInitialize", "200" }, { "mstest:timeout:classCleanup", "200" }, { "mstest:timeout:testInitialize", "100" }, { "mstest:timeout:testCleanup", "100" }, { "mstest:timeout:globalTestInitialize", "150" }, { "mstest:timeout:globalTestCleanup", "150" }, { "mstest:timeout:test", "60" }, { "mstest:timeout:useCooperativeCancellation", "true" }, { "mstest:parallelism:enabled", "true" }, { "mstest:parallelism:workers", "4" }, { "mstest:parallelism:scope", "class" }, { "mstest:execution:mapInconclusiveToFailed", "true" }, { "mstest:execution:mapNotRunnableToFailed", "true" }, { "mstest:execution:treatDiscoveryWarningsAsErrors", "true" }, { "mstest:execution:considerEmptyDataSourceAsInconclusive", "true" }, { "mstest:execution:orderTestsByNameInClass", "true" }, { "mstest:execution:randomizeTestOrder", "true" }, { "mstest:execution:randomTestOrderSeed", "-12345" }, { "mstest:execution:captureAssertionFailureDiagnostics", "true" }, { "mstest:output:captureTrace", "true" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OrderTestsByNameInClass.Should().BeTrue(); settings.RandomizeTestOrder.Should().BeTrue(); settings.RandomTestOrderSeed.Should().Be(-12345); settings.CaptureAssertionFailureDiagnostics.Should().BeTrue(); settings.CaptureDebugTraces.Should().BeTrue(); settings.CooperativeCancellationTimeout.Should().BeTrue(); settings.MapInconclusiveToFailed.Should().BeTrue(); settings.MapNotRunnableToFailed.Should().BeTrue(); settings.TreatDiscoveryWarningsAsErrors.Should().BeTrue(); settings.ConsiderEmptyDataSourceAsInconclusive.Should().BeTrue(); settings.TestTimeout.Should().Be(60); settings.AssemblyInitializeTimeout.Should().Be(300); settings.AssemblyCleanupTimeout.Should().Be(300); settings.ClassInitializeTimeout.Should().Be(200); settings.ClassCleanupTimeout.Should().Be(200); settings.TestInitializeTimeout.Should().Be(100); settings.TestCleanupTimeout.Should().Be(100); settings.GlobalTestInitializeTimeout.Should().Be(150); settings.GlobalTestCleanupTimeout.Should().Be(150); settings.DisableParallelization.Should().BeFalse(); settings.ParallelizationWorkers.Should().Be(4); settings.ParallelizationScope.Should().Be(ExecutionScope.ClassLevel); } public void ConfigJson_OutputCaptureMode_AcceptsEnumValue() { // Arrange var configDictionary = new Dictionary { { "mstest:output:captureTrace", "Live" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Live); settings.CaptureDebugTraces.Should().BeTrue(); } public void ConfigJson_OutputCaptureMode_MapsLegacyBooleanFalseToNone() { // Arrange var configDictionary = new Dictionary { { "mstest:output:captureTrace", "false" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.None); settings.CaptureDebugTraces.Should().BeFalse(); } public void ConfigJson_OutputCaptureMode_RejectsNumericValue() { // Arrange var configDictionary = new Dictionary { { "mstest:output:captureTrace", "2" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert - numeric input must not be treated as an enum ordinal (2 -> Live); it is rejected and the default is kept. settings.OutputCaptureMode.Should().Be(TestOutputCaptureMode.Result); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, "Invalid value '2' for runsettings entry 'output:captureTrace', setting will be ignored."), Times.Once); } public void RunSettings_WithGlobalFixtureTimeouts_ValuesAreSetCorrectly() { string runSettingsXml = """ 150 250 """; MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsNameAlias, _mockMessageLogger.Object.ToAdapterMessageLogger())!; adapterSettings.GlobalTestInitializeTimeout.Should().Be(150); adapterSettings.GlobalTestCleanupTimeout.Should().Be(250); } public void GlobalFixtureTimeout_WhenDedicatedKeySet_IsUsedInsteadOfSharedTestKey() { var settings = new MSTestSettings(); MSTestSettings.SetSettingsFromConfig( BuildConfig(new() { { "mstest:timeout:testInitialize", "100" }, { "mstest:timeout:testCleanup", "200" }, { "mstest:timeout:globalTestInitialize", "1100" }, { "mstest:timeout:globalTestCleanup", "2200" }, }), _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); MSTestSettings.PopulateSettings(settings); // The dedicated global keys take precedence over the per-test initialize/cleanup keys. TimeoutInfo.FromFixtureSettings(FixtureKind.GlobalTestInitialize).Timeout.Should().Be(1100); TimeoutInfo.FromFixtureSettings(FixtureKind.GlobalTestCleanup).Timeout.Should().Be(2200); } public void GlobalFixtureTimeout_WhenDedicatedKeyNotSet_FallsBackToSharedTestKey() { var settings = new MSTestSettings(); MSTestSettings.SetSettingsFromConfig( BuildConfig(new() { { "mstest:timeout:testInitialize", "100" }, { "mstest:timeout:testCleanup", "200" }, }), _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); MSTestSettings.PopulateSettings(settings); // With no dedicated global key, global fixtures keep honoring the per-test initialize/cleanup keys. TimeoutInfo.FromFixtureSettings(FixtureKind.GlobalTestInitialize).Timeout.Should().Be(100); TimeoutInfo.FromFixtureSettings(FixtureKind.GlobalTestCleanup).Timeout.Should().Be(200); } private static IConfiguration BuildConfig(Dictionary values) { var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => values.TryGetValue(key, out string? value) ? value : null); return mockConfig.Object; } public void ConfigJson_WithDeprecatedFlatOrderTestsByNameInClassKey_ValueIsSetAndWarningIsEmitted() { // Arrange - using the deprecated flat key var configDictionary = new Dictionary { { "mstest:orderTestsByNameInClass", "true" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OrderTestsByNameInClass.Should().BeTrue(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, Resource.DeprecatedFlatOrderTestsByNameInClassKey), Times.Once); } public void ConfigJson_WithExecutionOrderTestsByNameInClassKey_ValueIsSetAndNoWarningIsEmitted() { // Arrange - using the new grouped key var configDictionary = new Dictionary { { "mstest:execution:orderTestsByNameInClass", "true" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OrderTestsByNameInClass.Should().BeTrue(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, Resource.DeprecatedFlatOrderTestsByNameInClassKey), Times.Never); } public void ConfigJson_WithBothOrderTestsByNameInClassKeys_NewExecutionKeyTakesPrecedenceAndNoWarningIsEmitted() { // Arrange - both the new grouped key and the deprecated flat key are present with conflicting values. // The new key must win and, since it is present, the deprecated key is ignored without a warning. var configDictionary = new Dictionary { { "mstest:orderTestsByNameInClass", "false" }, { "mstest:execution:orderTestsByNameInClass", "true" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.OrderTestsByNameInClass.Should().BeTrue(); _mockMessageLogger.Verify(lm => lm.SendMessage(TestMessageLevel.Warning, Resource.DeprecatedFlatOrderTestsByNameInClassKey), Times.Never); } public void ConfigJson_Parllelism_Enabled_True() => ConfigJson_Parllelism_Enabled_Core(true); public void ConfigJson_Parllelism_Enabled_False() => ConfigJson_Parllelism_Enabled_Core(false); private void ConfigJson_Parllelism_Enabled_Core(bool parallelismEnabled) { // Arrange - setting up valid configuration values var configDictionary = new Dictionary { { "mstest:parallelism:enabled", parallelismEnabled.ToString().ToLowerInvariant() }, { "mstest:parallelism:workers", "4" }, { "mstest:parallelism:scope", "class" }, }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.DisableParallelization.Should().Be(!parallelismEnabled); settings.ParallelizationWorkers.Should().Be(4); settings.ParallelizationScope.Should().Be(ExecutionScope.ClassLevel); } public void ConfigJson_WithValidValues_MethodScope() { // Arrange - setting up valid configuration values var configDictionary = new Dictionary { ["mstest:parallelism:scope"] = "method" }; var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => configDictionary.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); // Act MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); // Assert settings.ParallelizationScope.Should().Be(ExecutionScope.MethodLevel); } #endregion #region Dependency declaration configuration tests /// /// Parses a flattened key/value configuration - the shape the JSON provider produces - through the real /// settings parser, so these tests exercise the same indexer-only surface it sees at run time. /// private MSTestSettings ParseDependencies(Dictionary values) { var mockConfig = new Mock(); mockConfig.Setup(config => config[It.IsAny()]) .Returns((string key) => values.TryGetValue(key, out string? value) ? value : null); var settings = new MSTestSettings(); MSTestSettings.SetSettingsFromConfig(mockConfig.Object, _mockMessageLogger.Object.ToAdapterMessageLogger(), settings); return settings; } public void DependencyConfig_ChainsProduceAnEdgePerAdjacentPair() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:chains:0:0", "Ns.C.First" }, { "mstest:execution:dependencies:chains:0:1", "Ns.C.Second" }, { "mstest:execution:dependencies:chains:0:2", "Ns.C.Third" }, }); settings.DeclaredDependencies.Should().NotBeNull(); settings.DeclaredDependencies!.Length.Should().Be(2); settings.DeclaredDependencies[0].Dependent.Should().Be("Ns.C.Second"); settings.DeclaredDependencies[0].Prerequisite.Should().Be("Ns.C.First"); settings.DeclaredDependencies[1].Dependent.Should().Be("Ns.C.Third"); settings.DeclaredDependencies[1].Prerequisite.Should().Be("Ns.C.Second"); } public void DependencyConfig_AChainOfOneDeclaresNoEdge() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:chains:0:0", "Ns.C.Only" }, }); settings.DeclaredDependencies.Should().BeNull(); } public void DependencyConfig_AnEmptyChainIsWarnedAboutAndDoesNotTruncateTheRest() { // An empty element stores nothing under its index, which through the indexer is indistinguishable // from the end of the array - so without the lookahead every later chain would silently vanish. MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:chains:1:0", "Ns.C.First" }, { "mstest:execution:dependencies:chains:1:1", "Ns.C.Second" }, }); settings.DeclaredDependencies.Should().ContainSingle(); settings.DeclaredDependencies![0].Dependent.Should().Be("Ns.C.Second"); _mockMessageLogger.Verify( lm => lm.SendMessage(TestMessageLevel.Warning, It.Is(m => m.Contains("chain at index 0"))), Times.Once); } public void DependencyConfig_NodesCarryEveryPrerequisiteAndProceedOnFailure() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:nodes:0:test", "Ns.C.Audit" }, { "mstest:execution:dependencies:nodes:0:dependsOn:0", "Ns.C.One" }, { "mstest:execution:dependencies:nodes:0:dependsOn:1", "Ns.C.Two" }, { "mstest:execution:dependencies:nodes:0:proceedOnFailure", "true" }, }); settings.DeclaredDependencies!.Length.Should().Be(2); settings.DeclaredDependencies.Should().OnlyContain(d => d.Dependent == "Ns.C.Audit" && d.ProceedOnFailure); } public void DependencyConfig_ProceedOnFailureDefaultsToFalseAndWarnsWhenNotABoolean() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:nodes:0:test", "Ns.C.Dependent" }, { "mstest:execution:dependencies:nodes:0:dependsOn:0", "Ns.C.Prereq" }, { "mstest:execution:dependencies:nodes:0:proceedOnFailure", "maybe" }, }); // Fails closed: an unparseable value must not become "run anyway". settings.DeclaredDependencies!.Single().ProceedOnFailure.Should().BeFalse(); _mockMessageLogger.Verify( lm => lm.SendMessage(TestMessageLevel.Warning, It.Is(m => m.Contains("maybe"))), Times.Once); } public void DependencyConfig_ANodeWithoutPrerequisitesIsWarnedAbout() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:nodes:0:test", "Ns.C.Lonely" }, }); settings.DeclaredDependencies.Should().BeNull(); _mockMessageLogger.Verify( lm => lm.SendMessage(TestMessageLevel.Warning, It.Is(m => m.Contains("Ns.C.Lonely"))), Times.Once); } public void DependencyConfig_ANodeMissingItsTestIsWarnedAboutAndDoesNotTruncateTheRest() { MSTestSettings settings = ParseDependencies(new() { { "mstest:execution:dependencies:nodes:1:test", "Ns.C.Dependent" }, { "mstest:execution:dependencies:nodes:1:dependsOn:0", "Ns.C.Prereq" }, }); settings.DeclaredDependencies.Should().ContainSingle(); settings.DeclaredDependencies![0].Dependent.Should().Be("Ns.C.Dependent"); _mockMessageLogger.Verify( lm => lm.SendMessage(TestMessageLevel.Warning, It.Is(m => m.Contains("node at index 0"))), Times.Once); } public void DependencyConfig_WhenNoDependenciesAreDeclaredNothingIsSet() => ParseDependencies(new() { { "mstest:execution:randomizeTestOrder", "true" } }) .DeclaredDependencies.Should().BeNull(); #endregion }