// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; namespace MSTest.Acceptance.IntegrationTests; [TestClass] public sealed class ConfigurationSettingsTests : AcceptanceTestBase { [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task TestConfigJson_AndRunSettingsWithoutMstest_OverrideRunConfigration(string tfm) { var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--settings my.runsettings", cancellationToken: TestContext.CancellationToken); // Assert testHostResult.AssertExitCodeIs(ExitCode.Success); } [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task TestConfigJson_WithoutRunSettings_BuildSuccess(string tfm) { var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); // Assert testHostResult.AssertExitCodeIs(ExitCode.Success); } [TestMethod] public async Task TestWithConfigFromCommandLineWithMapInconclusiveToFailedIsTrue() { var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); TestHostResult testHostResult = await testHost.ExecuteAsync( "--config-file dummyconfigfile_map.json", environmentVariables: new() { ["TestWithConfigFromCommandLine"] = "true", }, cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.AtLeastOneTestFailed); testHostResult.AssertOutputContainsSummary(failed: 1, passed: 1, skipped: 0); } [TestMethod] public async Task TestWithConfigFromCommandLineWithMapInconclusiveToFailedIsFalse() { var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); TestHostResult testHostResult = await testHost.ExecuteAsync( "--config-file dummyconfigfile_doNotMap.json", environmentVariables: new() { ["TestWithConfigFromCommandLine"] = "true", }, cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.Success); testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 1); } [TestMethod] public async Task TestWithConfigFromCommandLineWithNonExistingFile() { var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent); TestHostResult testHostResult = await testHost.ExecuteAsync( "--config-file dummyconfigfile_not_existing_file.json", environmentVariables: new() { ["TestWithConfigFromCommandLine"] = "true", }, cancellationToken: TestContext.CancellationToken); testHostResult.AssertStandardErrorContains("FileNotFoundException"); testHostResult.AssertStandardErrorContains("dummyconfigfile_not_existing_file.json"); } public sealed class TestAssetFixture() : TestAssetFixtureBase() { public const string ProjectName = "ConfigurationSettings"; public string ProjectPath => GetAssetPath(ProjectName); public override (string ID, string Name, string Code) GetAssetsToGenerate() => (ProjectName, ProjectName, SourceCode .PatchTargetFrameworks(TargetFrameworks.All) .PatchCodeWithReplace("$ProjectName$", ProjectName) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$AppendSettings$", string.Empty)); private const string SourceCode = """ #file $ProjectName$.csproj Exe true $TargetFrameworks$ Preview PreserveNewest Always Always Always #file my.runsettings true STA true $AppendSettings$ #file dummyconfigfile_map.json { "mstest": { "execution": { "mapInconclusiveToFailed": true, }, } } #file dummyconfigfile_doNotMap.json { "mstest": { "execution": { "mapInconclusiveToFailed": false, }, } } #file $ProjectName$.testconfig.json { "mstest": { "timeout": { "assemblyInitialize": 300000, "assemblyCleanup": 300000, "classInitialize": 200000, "classCleanup": 200000, "testInitialize": 100000, "testCleanup": 100000, "test": 60000, "useCooperativeCancellation": true }, "parallelism": { "enabled": true, "workers": 4, "scope": "class" }, "execution": { "collectSourceInformation": true, "executionApartmentState": "MTA", "disableAppDomain": false, "mapInconclusiveToFailed": true, "mapNotRunnableToFailed": true, "treatDiscoveryWarningsAsErrors": true, "considerEmptyDataSourceAsInconclusive": true, "orderTestsByNameInClass": true }, "deployment": { "enabled": true, "deployTestSourceDependencies": false, "deleteDeploymentDirectoryAfterTestRunIsComplete": true }, "assemblyResolution": [ { "path": "C:\\project\\dependencies", "includeSubDirectories": "true" }, { "path": "C:\\project\\libs", "includeSubDirectories": "true" }, { "path": "C:\\project\\plugins", "includeSubDirectories": "true" } ] } } #file UnitTest1.cs using System; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] public class UnitTest1 { [TestMethod] public void TestMethod() { } [TestMethod] public void TestWithConfigFromCommandLine() { if (Environment.GetEnvironmentVariable("TestWithConfigFromCommandLine") == "true") { Assert.Inconclusive("Inconclusive TestWithConfigFromCommandLine"); } } } """; } public TestContext TestContext { get; set; } }