// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; namespace MSTest.Acceptance.IntegrationTests; [TestClass] public class HelpInfoTests : AcceptanceTestBase { private const string AssetName = "HelpInfo"; [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task Help_WhenMSTestExtensionRegistered_OutputHelpContentOfRegisteredExtension(string tfm) { var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--help", cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.Success); string wildcardMatchPattern = $""" MSTest v{MSTestVersion} (UTC *) [* - *] Usage {AssetName}* [option providers] [extension option providers] Execute a .NET Test Application. Options: --ansi Control whether ANSI escape characters are emitted. Valid values are 'auto' (default), 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). 'on' forces ANSI escape codes (including cursor movement) even when stdout is redirected; pair it with --progress off if you only want colors. When both --ansi and --no-ansi are provided, --ansi wins. --config-file Specifies a testconfig.json file. --coverage-threshold-branch Specifies the minimum overall branch coverage percentage, from 0 to 100. Requires non-empty coverage data from a compatible collector. --coverage-threshold-line Specifies the minimum overall line coverage percentage, from 0 to 100. Requires non-empty coverage data from a compatible collector. --debug Allows to pause execution in order to attach to the process for debug purposes. --diagnostic Enable the diagnostic logging. The default log level is 'Trace'. The file will be written in the output directory with the name log_[yyMMddHHmmssfff].diag. --diagnostic-file-prefix Replace '[log]_.' with the specified log file name prefix. --diagnostic-output-directory Output directory of the diagnostic logging. If not specified the file will be generated inside the default 'TestResults' directory. --diagnostic-synchronous-write Force the built-in file logger to write the log synchronously. Useful for scenario where you don't want to lose any log (i.e. in case of crash). Note that this is slowing down the test execution. --diagnostic-verbosity Define the level of the verbosity for the --diagnostic. The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', and 'Critical'. --enable-dynamic-extensions Enable loading test platform extensions declared by '*.testingplatformextensions.json' manifests found next to the test application. Disabled by default. --exit-on-process-exit Exit the test process if dependent process exits. PID must be provided. --filter-uid Provides a list of test node UIDs to filter by. --help Show the command line help. --ignore-exit-code Do not report a non-successful exit value for the specified exit codes. For example, '--ignore-exit-code 8;9' ignores exit codes 8 and 9 and returns 0 in those cases. For more information about exit codes, see https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes. --info Display .NET test application information. --list-tests List available tests. Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --no-ansi Disable outputting ANSI escape characters to screen. --no-progress [Deprecated, use '--progress off' instead] Disable reporting progress to screen. --output Preset the per-test result blocks shown in the terminal. Valid values are 'Minimal', 'Normal', 'Detailed'. Default is 'Normal'. Use '--show-test-results' for precise outcome selection. --progress Control whether progress is reported to screen. Valid values are 'auto' (default), 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). 'auto' shows progress unless the terminal cannot update in place (for example with --no-ansi or in CI). This option takes precedence over the deprecated --no-progress flag. --results-directory The directory where the test results are going to be placed. If the specified directory doesn't exist, it's created. The default is TestResults in the directory that contains the test application. --show-flaky-tests Control whether tests that failed at least once but eventually passed after a retry are listed in the run summary. Valid values are 'on' (the default, also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Passing the option without a value turns it on. --show-slowest-tests Show the specified number of slowest tests (by reported execution duration) in the run summary. Expects a positive integer. --show-stderr Determines when to show captured error output of a test. Valid values are 'All', 'Failed', 'None'. Default is 'All' (or 'Failed' when an LLM/AI agent environment is detected). --show-stdout Determines when to show captured standard output of a test. Valid values are 'All', 'Failed', 'None'. Default is 'All' (or 'Failed' when an LLM/AI agent environment is detected). --show-test-results Selects which test outcomes render a per-test result block (with its informative details, stack trace, and captured output) in the terminal. Valid values are 'passed', 'failed' (also covers errored, timed out, and canceled tests), 'skipped', 'all', and 'none'. Combine multiple values as a comma- or space-separated list, or by repeating the option; 'all' and 'none' cannot be combined with any other value. Default is 'failed' when '--output' is 'Minimal', 'failed' and 'skipped' when it is 'Normal' or omitted, and 'all' when it is 'Detailed'. An explicit '--show-test-results' always takes precedence over '--output', regardless of the order they are passed in. --timeout A global test execution timeout. Takes one argument as a time value with an explicit unit suffix. Accepted suffixes are 'ms'/'mil(s)'/'millisecond(s)', 's'/'sec(s)'/'second(s)', 'm'/'min(s)'/'minute(s)', 'h'/'hour(s)', and 'd'/'day(s)', e.g. '500ms', '5400s', '90m', '1.5h', '1d'. --zero-tests-policy Specifies how a run that executed no tests is treated. Valid values are 'allow-skipped' (the default) which counts skipped tests as run, so only a run where no test was found at all fails with exit code 8, and 'strict' which treats skipped tests as not run, so a run where every test was skipped (or no test was found) fails with exit code 8. Extension options: --filter Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. --maximum-failed-tests Specifies a maximum number of test failures that, when exceeded, will abort the test run. --settings The path, relative or absolute, to the .runsettings file. For more information and examples on how to configure test run, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#the-runsettings-file. --test-parameter Specify or override a key-value pair parameter. For more information and examples, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#testrunparameters. """; testHostResult.AssertOutputMatchesLines(wildcardMatchPattern); } [TestMethod] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task Info_WhenMSTestExtensionRegistered_OutputInfoContentOfRegisteredExtension(string tfm) { var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); TestHostResult testHostResult = await testHost.ExecuteAsync("--info", cancellationToken: TestContext.CancellationToken); testHostResult.AssertExitCodeIs(ExitCode.Success); string output = $""" MSTestExtension Name: MSTest Version: {MSTestVersion} Description: MSTest Framework for Microsoft Testing Platform. Options: --settings Arity: 1 Hidden: False Description: The path, relative or absolute, to the .runsettings file. For more information and examples on how to configure test run, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#the-runsettings-file. --filter Arity: 1 Hidden: False Description: Filters tests using the given expression. For more information, see the Filter option details section. For more information and examples on how to use selective unit test filtering, see https://learn.microsoft.com/dotnet/core/testing/selective-unit-tests. --test-parameter Arity: 1..N Hidden: False Description: Specify or override a key-value pair parameter. For more information and examples, see https://learn.microsoft.com/visualstudio/test/configure-unit-tests-by-using-a-dot-runsettings-file#testrunparameters. --maximum-failed-tests Arity: 1 Hidden: False Description: Specifies a maximum number of test failures that, when exceeded, will abort the test run. """; testHostResult.AssertOutputContains(output); } public sealed class TestAssetFixture() : TestAssetFixtureBase() { public string TargetAssetPath => GetAssetPath(AssetName); public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, SourceCode .PatchTargetFrameworks(TargetFrameworks.All) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); private const string SourceCode = """ #file HelpInfo.csproj Exe true $TargetFrameworks$ #file UnitTest1.cs using System; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] public class TestClass { [TestMethod] public void Test() {} } """; } public TestContext TestContext { get; set; } }