// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Configurations; using Microsoft.Testing.Platform.Extensions.CommandLine; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Services; using Moq; namespace Microsoft.Testing.Platform.UnitTests; /// /// Validates the JSON-aware enumeration backing the unified read model: /// / /// JsonConfigurationProvider.EnumerateCommandLineOptions, plus the corresponding /// validator passes in that prevent runtime crashes /// when a testconfig.json file declares unknown options, supplies the wrong arity, or trips a /// provider's argument validator. /// [TestClass] public sealed class JsonCommandLineOptionsTests { // --------------------------------------------------------------------- // AggregatedConfiguration.EnumerateJsonCommandLineOptions // --------------------------------------------------------------------- [TestMethod] public void EnumerateJsonCommandLineOptions_NoJsonSource_ReturnsEmpty() { AggregatedConfiguration configuration = new( [], new CurrentTestApplicationModuleInfo(new SystemEnvironment(), new SystemProcessHandler()), new Mock().Object, new SystemEnvironment(), CommandLineParseResult.Empty); IReadOnlyList entries = configuration.EnumerateJsonCommandLineOptions(); Assert.IsEmpty(entries); } // --------------------------------------------------------------------- // JsonConfigurationProvider.EnumerateCommandLineOptions schema cases // --------------------------------------------------------------------- [TestMethod] public async Task EnumerateCommandLineOptions_ScalarString_YieldsSingleArgumentEntry() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"timeout\": \"30s\"}}"); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("timeout", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.AreSequenceEqual(["30s"], entry.Arguments.ToArray()); } [TestMethod] public async Task EnumerateCommandLineOptions_BooleanTrue_YieldsZeroArityEntry() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"no-banner\": true}}"); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("no-banner", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.IsEmpty(entry.Arguments); } [TestMethod] public async Task EnumerateCommandLineOptions_BooleanFalse_YieldsDisabledEntry() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"no-banner\": false}}"); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("no-banner", entry.OptionName); Assert.IsTrue(entry.IsDisabled); Assert.IsEmpty(entry.Arguments); } [TestMethod] public async Task EnumerateCommandLineOptions_Array_YieldsAllArguments() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"filter-uid\": [\"a\", \"b\", \"c\"]}}"); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("filter-uid", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.AreSequenceEqual(["a", "b", "c"], entry.Arguments.ToArray()); } [TestMethod] public async Task EnumerateCommandLineOptions_EmptyArray_IsSkipped() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"filter-uid\": []}}"); Assert.IsEmpty(entries); } [TestMethod] public async Task EnumerateCommandLineOptions_EmptyObject_IsRejected() { FormatException ex = await Assert.ThrowsExactlyAsync( () => EnumerateAsync("{\"commandLineOptions\": {\"foo\": {}}}")); Assert.Contains("foo", ex.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_NestedObject_IsRejected() { FormatException ex = await Assert.ThrowsExactlyAsync( () => EnumerateAsync("{\"commandLineOptions\": {\"foo\": {\"nested\": \"value\"}}}")); Assert.Contains("foo", ex.Message); // The error message should render the entry name relative to the section // (e.g. 'foo') rather than the redundant flattened key ('commandLineOptions:foo'). Assert.DoesNotContain("'commandLineOptions:foo'", ex.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_ArrayOfObjects_IsRejected() { FormatException ex = await Assert.ThrowsExactlyAsync( () => EnumerateAsync("{\"commandLineOptions\": {\"foo\": [{\"k\":\"v\"}]}}")); Assert.Contains("foo", ex.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_SectionIsScalar_IsRejected() { FormatException ex = await Assert.ThrowsExactlyAsync( () => EnumerateAsync("{\"commandLineOptions\": \"not-an-object\"}")); Assert.Contains("commandLineOptions", ex.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_SectionIsArray_IsRejected() { FormatException ex = await Assert.ThrowsExactlyAsync( () => EnumerateAsync("{\"commandLineOptions\": [\"a\",\"b\"]}")); Assert.Contains("commandLineOptions", ex.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_SectionAbsent_ReturnsEmpty() { IReadOnlyList entries = await EnumerateAsync("{}"); Assert.IsEmpty(entries); } [TestMethod] public async Task EnumerateCommandLineOptions_MixedEntries_AllReturned() { IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"timeout\": \"30s\", \"no-banner\": true, \"hangdump\": false, \"filter-uid\": [\"a\",\"b\"]}}"); Assert.HasCount(4, entries); JsonCommandLineOptionEntry timeout = entries.Single(e => e.OptionName == "timeout"); Assert.AreSequenceEqual(["30s"], timeout.Arguments.ToArray()); Assert.IsFalse(timeout.IsDisabled); JsonCommandLineOptionEntry noBanner = entries.Single(e => e.OptionName == "no-banner"); Assert.IsEmpty(noBanner.Arguments); Assert.IsFalse(noBanner.IsDisabled); JsonCommandLineOptionEntry hangdump = entries.Single(e => e.OptionName == "hangdump"); Assert.IsTrue(hangdump.IsDisabled); JsonCommandLineOptionEntry filterUid = entries.Single(e => e.OptionName == "filter-uid"); Assert.AreSequenceEqual(["a", "b"], filterUid.Arguments.ToArray()); } [TestMethod] public async Task EnumerateCommandLineOptionDefaults_ScalarsAndArrays_AreArguments() { AggregatedConfiguration configuration = await BuildAggregatedAsync( """ { "commandLineOptionDefaults": { "timeout": "30s", "boolean-value": false, "filter-uid": ["a", "b"] } } """); IReadOnlyList entries = configuration.EnumerateJsonCommandLineOptionDefaults(); Assert.HasCount(3, entries); Assert.AreSequenceEqual(["30s"], entries.Single(x => x.OptionName == "timeout").Arguments.ToArray()); Assert.AreSequenceEqual(["False"], entries.Single(x => x.OptionName == "boolean-value").Arguments.ToArray()); Assert.AreSequenceEqual(["a", "b"], entries.Single(x => x.OptionName == "filter-uid").Arguments.ToArray()); Assert.IsTrue(configuration.TryGetCommandLineOptionDefaultFromProviders("boolean-value", out string[] booleanArguments)); Assert.AreSequenceEqual(["False"], booleanArguments); Assert.IsFalse(configuration.TryGetCommandLineOptionFromProviders("timeout", out bool isSet, out _)); Assert.IsFalse(isSet); } [TestMethod] [DataRow("null")] [DataRow("[]")] public async Task EnumerateCommandLineOptionDefaults_MissingValue_IsRejected(string value) { AggregatedConfiguration configuration = await BuildAggregatedAsync( "{\"commandLineOptionDefaults\": {\"timeout\": " + value + "}}"); FormatException exception = Assert.ThrowsExactly( configuration.EnumerateJsonCommandLineOptionDefaults); Assert.Contains("timeout", exception.Message); Assert.Contains("non-null scalar value or a non-empty array", exception.Message); } [TestMethod] public async Task EnumerateCommandLineOptionDefaults_ArrayWithNullElement_IsRejected() { // A JSON null array element must be rejected even though both JSON parsers (System.Text.Json and, // on net462/netstandard2.0, Jsonite) flatten it into a non-null placeholder string ("" or "null" // respectively) that looks like a real scalar. The parser separately preserves the element's raw // JSON token so this case can be detected consistently across both parsers. AggregatedConfiguration configuration = await BuildAggregatedAsync( "{\"commandLineOptionDefaults\": {\"filter-uid\": [null]}}"); FormatException exception = Assert.ThrowsExactly( configuration.EnumerateJsonCommandLineOptionDefaults); Assert.Contains("filter-uid", exception.Message); Assert.Contains("non-null scalar value or a non-empty array", exception.Message); } [TestMethod] public async Task EnumerateCommandLineOptionDefaults_MixedArrayWithNullElement_IsRejected() { // A null element must be rejected even when mixed with otherwise-valid scalar entries in the same // array, and regardless of its position. AggregatedConfiguration configuration = await BuildAggregatedAsync( "{\"commandLineOptionDefaults\": {\"filter-uid\": [\"a\", null, \"b\"]}}"); FormatException exception = Assert.ThrowsExactly( configuration.EnumerateJsonCommandLineOptionDefaults); Assert.Contains("filter-uid", exception.Message); Assert.Contains("non-null scalar value or a non-empty array", exception.Message); } [TestMethod] public async Task EnumerateCommandLineOptions_ArrayWithNullElement_PreservesEstablishedBehavior() { // Unlike commandLineOptionDefaults, commandLineOptions does not require every array element to be // non-null; this locks in the pre-existing (per-JSON-parser) flattening of a null element so this // fix does not change it. System.Text.Json flattens a null array element to an empty string, while // the Jsonite parser (net462/netstandard2.0) flattens it to the literal "null" string. IReadOnlyList entries = await EnumerateAsync( "{\"commandLineOptions\": {\"filter-uid\": [\"a\", null, \"b\"]}}"); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("filter-uid", entry.OptionName); #if NETFRAMEWORK Assert.AreSequenceEqual(["a", "null", "b"], entry.Arguments.ToArray()); #else Assert.AreSequenceEqual(["a", string.Empty, "b"], entry.Arguments.ToArray()); #endif } [TestMethod] public async Task ExplicitJsonDisable_SuppressesConfiguredDefault() { AggregatedConfiguration configuration = await BuildAggregatedAsync( """ { "commandLineOptions": { "optional-value": false }, "commandLineOptionDefaults": { "optional-value": "fallback" } } """); Assert.IsFalse(configuration.TryGetCommandLineOptionArgumentsOrDefault("optional-value", out string[]? arguments)); Assert.IsNull(arguments); } // --------------------------------------------------------------------- // CommandLineOptionsValidator JSON-aware passes // --------------------------------------------------------------------- [TestMethod] public async Task Validator_UnknownOptionInJson_FailsWithJsonPrefix() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("not-a-real-option", [], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("not-a-real-option", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonArityTooFew_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", [], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("timeout", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonArityTooMany_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", ["30s", "60s"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("timeout", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonZeroArityWithArguments_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("flag", "desc", ArgumentArity.Zero, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("flag", ["unexpected"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("flag", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonDisabledEntry_Skipped() { // "foo": false should NOT trigger arity errors even though arity.Min > 0 — disabled // entries convey "the option is not set" and have no arguments to validate. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", [], isDisabled: true)]); Assert.IsTrue(result.IsValid, result.ErrorMessage); } [TestMethod] public async Task Validator_JsonPerArgValidatorFailure_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false), validateOptionArgumentsAsync: (option, args) => Task.FromResult(args[0] == "bad" ? ValidationResult.Invalid("bad value") : ValidationResult.Valid())); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", ["bad"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("bad value", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonDefaultPerArgValidatorFailure_FailsWithDefaultsPrefix() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false), validateOptionArgumentsAsync: (option, args) => Task.FromResult(args[0] == "bad" ? ValidationResult.Invalid("bad value") : ValidationResult.Valid())); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptionDefaults: [new JsonCommandLineOptionEntry("timeout", ["bad"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("bad value", result.ErrorMessage); Assert.Contains("commandLineOptionDefaults", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonDefaultForZeroArityOption_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("no-banner", "desc", ArgumentArity.Zero, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptionDefaults: [new JsonCommandLineOptionEntry("no-banner", ["true"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("commandLineOptionDefaults", result.ErrorMessage); Assert.Contains("no-banner", result.ErrorMessage); } [TestMethod] public async Task Validator_UnknownJsonDefault_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptionDefaults: [new JsonCommandLineOptionEntry("timoeut", ["30s"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("commandLineOptionDefaults", result.ErrorMessage); Assert.Contains("timoeut", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonArityFailure_DoesNotInvokePerArgValidator() { // Per-arg validators may index into args (e.g., args[0] for ExactlyOne). If we ran the // per-arg pass on a JSON entry that already failed arity (Count != 1), it would crash. The // validator must skip per-arg checks for entries that fail arity. bool perArgInvoked = false; ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false), validateOptionArgumentsAsync: (option, args) => { perArgInvoked = true; _ = args[0]; // would IndexOutOfRange if invoked with empty args return Task.FromResult(ValidationResult.Valid()); }); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", [], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.IsFalse(perArgInvoked, "Per-arg validator must not run for entries that already fail arity."); } [TestMethod] public async Task Validator_CliShadowsJsonTypo_TypoStillReported() { // A JSON typo silently overridden by the CLI is still a typo the user wants to know about. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); CommandLineParseResult cliParseResult = new( toolName: null, options: [new CommandLineParseOption("timeout", ["30s"])], errors: []); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( cliParseResult, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timoeut", ["30s"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("timoeut", result.ErrorMessage); } [TestMethod] public async Task Validator_NullJsonList_BackwardCompatibleNoErrors() { // Existing call sites that pass no JSON list (default null) must continue to pass. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object); Assert.IsTrue(result.IsValid, result.ErrorMessage); } // --------------------------------------------------------------------- // Bootstrap-only options must not appear under testconfig.json's // "commandLineOptions". They are read off CommandLineParseResult before // IConfiguration is built (see TestApplication.cs and // JsonConfigurationProvider.cs), so a JSON entry is silently ignored. // The validator surfaces this with a clear error instead. // --------------------------------------------------------------------- [TestMethod] [DataRow(PlatformCommandLineProvider.ConfigFileOptionKey)] [DataRow(PlatformCommandLineProvider.DiagnosticOptionKey)] [DataRow(PlatformCommandLineProvider.DiagnosticOutputDirectoryOptionKey)] [DataRow(PlatformCommandLineProvider.DiagnosticOutputFilePrefixOptionKey)] [DataRow(PlatformCommandLineProvider.DiagnosticVerbosityOptionKey)] [DataRow(PlatformCommandLineProvider.DiagnosticFileLoggerSynchronousWriteOptionKey)] [DataRow(PlatformCommandLineProvider.EnableDynamicExtensionsOptionKey)] public async Task Validator_JsonBootstrapOnlyOption_Fails(string optionName) { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(optionName, "desc", ArgumentArity.ZeroOrOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry(optionName, ["value"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains(optionName, result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); Assert.Contains("bootstrap", result.ErrorMessage, StringComparison.OrdinalIgnoreCase); } [TestMethod] public async Task Validator_JsonBootstrapOnlyOption_CaseInsensitive_Fails() { // testconfig.json keys are case-insensitive everywhere else; the bootstrap-only check // must match regardless of casing so "Config-File" is flagged the same as "config-file". ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(PlatformCommandLineProvider.ConfigFileOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("Config-File", ["other.json"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("Config-File", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_JsonDefaultBootstrapOnlyOption_Fails() { ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(PlatformCommandLineProvider.ConfigFileOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptionDefaults: [ new JsonCommandLineOptionEntry(PlatformCommandLineProvider.ConfigFileOptionKey, ["other.json"], isDisabled: false), ]); Assert.IsFalse(result.IsValid); Assert.Contains(PlatformCommandLineProvider.ConfigFileOptionKey, result.ErrorMessage); Assert.Contains("commandLineOptionDefaults", result.ErrorMessage); Assert.Contains("bootstrap", result.ErrorMessage, StringComparison.OrdinalIgnoreCase); } [TestMethod] public async Task Validator_JsonBootstrapOnlyOption_DisabledEntry_StillFails() { // "config-file": false in testconfig.json is just as misleading as setting it to a value: // either way the option is read pre-config and the JSON entry has no effect. Flag both. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(PlatformCommandLineProvider.ConfigFileOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry(PlatformCommandLineProvider.ConfigFileOptionKey, [], isDisabled: true)]); Assert.IsFalse(result.IsValid); Assert.Contains(PlatformCommandLineProvider.ConfigFileOptionKey, result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_CliBootstrapOnlyOption_Allowed() { // Setting bootstrap-only options on the CLI is the supported path and must remain valid. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(PlatformCommandLineProvider.ConfigFileOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false)); CommandLineParseResult cliParseResult = new( toolName: null, options: [new CommandLineParseOption(PlatformCommandLineProvider.ConfigFileOptionKey, ["other.json"])], errors: []); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( cliParseResult, [provider], [], new Mock().Object, jsonCommandLineOptions: null); Assert.IsTrue(result.IsValid, result.ErrorMessage); } [TestMethod] public async Task Validator_JsonNonBootstrapOption_Allowed() { // Sanity check: non-bootstrap-only options (e.g. timeout) continue to be honored from // testconfig.json. The bootstrap-only check must not regress general JSON support. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("timeout", ["30s"], isDisabled: false)]); Assert.IsTrue(result.IsValid, result.ErrorMessage); } [TestMethod] public async Task Validator_JsonMultipleBootstrapOnlyOptions_AllReported() { // The validator aggregates bootstrap-only violations so users see every offending entry // in one pass rather than fixing them one by one across multiple runs. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption(PlatformCommandLineProvider.ConfigFileOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false), new CommandLineOption(PlatformCommandLineProvider.DiagnosticOptionKey, "desc", ArgumentArity.Zero, isHidden: false), new CommandLineOption(PlatformCommandLineProvider.DiagnosticVerbosityOptionKey, "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [ new JsonCommandLineOptionEntry(PlatformCommandLineProvider.ConfigFileOptionKey, ["other.json"], isDisabled: false), new JsonCommandLineOptionEntry(PlatformCommandLineProvider.DiagnosticOptionKey, [], isDisabled: false), new JsonCommandLineOptionEntry(PlatformCommandLineProvider.DiagnosticVerbosityOptionKey, ["trace"], isDisabled: false), ]); Assert.IsFalse(result.IsValid); Assert.Contains(PlatformCommandLineProvider.ConfigFileOptionKey, result.ErrorMessage); Assert.Contains(PlatformCommandLineProvider.DiagnosticOptionKey, result.ErrorMessage); Assert.Contains(PlatformCommandLineProvider.DiagnosticVerbosityOptionKey, result.ErrorMessage); } [TestMethod] public async Task Validator_JsonUnknownOptionCaseInsensitive_Accepted() { // testconfig.json keys are case-insensitive elsewhere in the platform; "Timeout" must // resolve to the registered "timeout" option rather than be reported as unknown. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("Timeout", ["30s"], isDisabled: false)]); Assert.IsTrue(result.IsValid, result.ErrorMessage); } [TestMethod] public async Task EnumerateCommandLineOptions_SectionNameCaseInsensitive_Honored() { // The JSON parser's internal dictionary is OrdinalIgnoreCase, so a top-level key spelled // "CommandLineOptions" must still surface under the "commandLineOptions" section. IReadOnlyList entries = await EnumerateAsync( """{"CommandLineOptions": {"timeout": "30s"}}"""); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(entries); Assert.AreEqual("timeout", entry.OptionName); Assert.AreEqual("30s", Assert.ContainsSingle(entry.Arguments)); } [TestMethod] public async Task Validator_EmptyOptionNameInJson_FailsWithJsonPrefix() { // Defensive: a user typo such as { "commandLineOptions": { "": "value" } } surfaces as // an unknown-option error rather than crashing inside the validator. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry(string.Empty, ["value"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("testconfig.json", result.ErrorMessage); } [TestMethod] public async Task Validator_DuplicateOptionNamesDifferingByCase_FailsGracefully() { // Two extension providers register the same option with different casing. Pre-fix the // default Ordinal comparison in ValidateOptionsAreNotDuplicated treated them as distinct // names and both were silently accepted, even though every downstream lookup // (TryGetCommandLineOptionFromProviders, testconfig.json) is case-insensitive. The // OrdinalIgnoreCase fix surfaces this as a clean ValidationResult.Invalid up front. ICommandLineOptionsProvider providerA = new TestProvider( new CommandLineOption("Timeout", "desc-A", ArgumentArity.ExactlyOne, isHidden: false)); ICommandLineOptionsProvider providerB = new TestProvider( new CommandLineOption("timeout", "desc-B", ArgumentArity.ExactlyOne, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, systemCommandLineOptionsProviders: [], extensionCommandLineOptionsProviders: [providerA, providerB], new Mock().Object); Assert.IsFalse(result.IsValid); // Pin the diagnostic to the duplicate-options pass (rather than any failure) by asserting // on the unique wording from that pass plus both provider display names appearing in the // error so a future refactor that swaps which pass surfaces the failure will be caught. Assert.Contains("declared by multiple", result.ErrorMessage); Assert.Contains(providerA.DisplayName, result.ErrorMessage); Assert.Contains(providerB.DisplayName, result.ErrorMessage); } [TestMethod] public async Task Validator_DuplicateOptionNamesAcrossSystemAndSystem_FailsGracefully() { // Symmetric case: two system providers registering the same name with different casing // must also fail through the duplicate-options pass rather than throw later. ICommandLineOptionsProvider systemA = new TestProvider( new CommandLineOption("Verbose", "desc-A", ArgumentArity.Zero, isHidden: false)); ICommandLineOptionsProvider systemB = new TestProvider( new CommandLineOption("verbose", "desc-B", ArgumentArity.Zero, isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, systemCommandLineOptionsProviders: [systemA, systemB], extensionCommandLineOptionsProviders: [], new Mock().Object); Assert.IsFalse(result.IsValid); Assert.Contains("declared by multiple", result.ErrorMessage); Assert.Contains(systemA.DisplayName, result.ErrorMessage); Assert.Contains(systemB.DisplayName, result.ErrorMessage); } [TestMethod] public async Task Validator_JsonEntryWithTooFewArguments_FailsArityCheck() { // A JSON entry whose argument count is below the option's minimum arity must surface // through the arity validation pass with a testconfig.json-prefixed error rather than // crashing later inside the provider. ICommandLineOptionsProvider provider = new TestProvider( new CommandLineOption("pair", "desc", new ArgumentArity(2, 2), isHidden: false)); ValidationResult result = await CommandLineOptionsValidator.ValidateAsync( CommandLineParseResult.Empty, [provider], [], new Mock().Object, jsonCommandLineOptions: [new JsonCommandLineOptionEntry("pair", ["only-one"], isDisabled: false)]); Assert.IsFalse(result.IsValid); Assert.Contains("pair", result.ErrorMessage); Assert.Contains("testconfig.json", result.ErrorMessage); } // --------------------------------------------------------------------- // NormalizeJsonCommandLineOptionScalars (issue #8830 — scalar/bool ambiguity) // --------------------------------------------------------------------- [TestMethod] public async Task NormalizeScalars_ArgBearingOption_BooleanStringTrue_BecomesArgumentValue() { // The pre-#8830 behavior treated "timeout": "true" as a presence marker because the // scalar value parsed as a bool. After normalization the value must be surfaced as the // first argument, both through EnumerateJsonCommandLineOptions and through the unified // ICommandLineOptions lookup. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"timeout\": \"true\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("timeout", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.AreSequenceEqual(["true"], entry.Arguments.ToArray()); Assert.IsTrue(aggregated.TryGetCommandLineOptionFromProviders("timeout", out bool isSet, out string[] arguments)); Assert.IsTrue(isSet); Assert.AreSequenceEqual(["true"], arguments); } [TestMethod] public async Task NormalizeScalars_ArgBearingOption_BooleanStringFalse_BecomesArgumentValue() { // Symmetric to the "true" case: "false" must not be interpreted as an explicit disable for // an arg-bearing option. Without normalization the unified lookup would report isSet=false // and the option would silently appear unset to the rest of the platform. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"timeout\": \"false\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("timeout", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.AreSequenceEqual(["false"], entry.Arguments.ToArray()); Assert.IsTrue(aggregated.TryGetCommandLineOptionFromProviders("timeout", out bool isSet, out string[] arguments)); Assert.IsTrue(isSet); Assert.AreSequenceEqual(["false"], arguments); } [TestMethod] public async Task NormalizeScalars_ArgBearingOption_NonBoolScalar_RemainsAsArgument() { // Non-bool scalars already worked before #8830 thanks to the second branch in // TryGetCommandLineOptionFromProviders; normalization must preserve this behavior while // moving the storage into the canonical indexed shape. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"timeout\": \"30s\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("timeout", entry.OptionName); Assert.AreSequenceEqual(["30s"], entry.Arguments.ToArray()); Assert.IsTrue(aggregated.TryGetCommandLineOptionFromProviders("timeout", out bool isSet, out string[] arguments)); Assert.IsTrue(isSet); Assert.AreSequenceEqual(["30s"], arguments); } [TestMethod] public async Task NormalizeScalars_ZeroArityOption_BooleanScalar_RemainsPresenceMarker() { // Zero-arity flags must keep their bool→presence-marker semantics. Normalization is a // no-op for them because their arity (Min == 0) signals that the bare key is meaningful // as-is. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"no-banner\": true}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("no-banner", "desc", ArgumentArity.Zero, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("no-banner", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.IsEmpty(entry.Arguments); Assert.IsTrue(aggregated.TryGetCommandLineOptionFromProviders("no-banner", out bool isSet, out string[] arguments)); Assert.IsTrue(isSet); Assert.IsEmpty(arguments); } [TestMethod] public async Task NormalizeScalars_OptionalArgOption_BooleanScalar_LeftUntouched() { // Min=0, Max>=1 (optional-arg) is genuinely ambiguous between a presence marker and a // scalar argument. We preserve the historical presence-marker interpretation; users that // really mean to pass "true"/"false" as a value must use the explicit array form. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"opt-arg\": \"true\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("opt-arg", "desc", ArgumentArity.ZeroOrOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("opt-arg", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.IsEmpty(entry.Arguments); } [TestMethod] public async Task NormalizeScalars_UnknownOption_LeftUntouched() { // Unknown options must not be rewritten — the validator's unknown-option pass needs to see // the original entry to emit a clear error referencing testconfig.json. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"unknown\": \"true\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("other", "desc", ArgumentArity.ExactlyOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreEqual("unknown", entry.OptionName); Assert.IsFalse(entry.IsDisabled); Assert.IsEmpty(entry.Arguments); } [TestMethod] public async Task NormalizeScalars_IndexedEntries_LeftUntouched() { // Array form is already in the canonical shape and must not be touched, including when an // option that takes multiple args is involved. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"filter-uid\": [\"a\",\"b\"]}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("filter-uid", "desc", ArgumentArity.OneOrMore, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreSequenceEqual(["a", "b"], entry.Arguments.ToArray()); Assert.IsTrue(aggregated.TryGetCommandLineOptionFromProviders("filter-uid", out bool isSet, out string[] arguments)); Assert.IsTrue(isSet); Assert.AreSequenceEqual(["a", "b"], arguments); } [TestMethod] public async Task NormalizeScalars_CaseInsensitive_RewritesByOptionName() { // testconfig.json keys are case-insensitive throughout the platform. A JSON entry "Timeout" // must be matched against the registry's "timeout" and normalized accordingly. AggregatedConfiguration aggregated = await BuildAggregatedAsync( "{\"commandLineOptions\": {\"Timeout\": \"true\"}}"); aggregated.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false))); JsonCommandLineOptionEntry entry = Assert.ContainsSingle(aggregated.EnumerateJsonCommandLineOptions()); Assert.AreSequenceEqual(["true"], entry.Arguments.ToArray()); } [TestMethod] public async Task NormalizeScalars_NoJsonSource_IsNoOp() { // The platform may build an AggregatedConfiguration without a JSON source registered; // normalization must degrade to a no-op rather than throw. AggregatedConfiguration configuration = new( [], new CurrentTestApplicationModuleInfo(new SystemEnvironment(), new SystemProcessHandler()), new Mock().Object, new SystemEnvironment(), CommandLineParseResult.Empty); configuration.NormalizeJsonCommandLineOptionScalars(OptionsByName( new CommandLineOption("timeout", "desc", ArgumentArity.ExactlyOne, isHidden: false))); Assert.IsEmpty(configuration.EnumerateJsonCommandLineOptions()); // The await keeps the test method asynchronous to match the surrounding suite style. await Task.CompletedTask; } // --------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------- private static async Task> EnumerateAsync(string json) { AggregatedConfiguration aggregated = await BuildAggregatedAsync(json); return aggregated.EnumerateJsonCommandLineOptions(); } private static async Task BuildAggregatedAsync(string json) { Mock fileSystem = new(); fileSystem.Setup(x => x.ExistFile(It.IsAny())).Returns(true); fileSystem.Setup(x => x.NewFileStream(It.IsAny(), FileMode.Open, FileAccess.Read)) .Returns(() => new ConfigurationManagerTests.MemoryFileStream(Encoding.UTF8.GetBytes(json))); CurrentTestApplicationModuleInfo testApplicationModuleInfo = new(new SystemEnvironment(), new SystemProcessHandler()); ConfigurationManager configurationManager = new(fileSystem.Object, testApplicationModuleInfo, new SystemEnvironment()); configurationManager.AddConfigurationSource(() => new JsonConfigurationSource(testApplicationModuleInfo, fileSystem.Object, null)); IConfiguration configuration = await configurationManager.BuildAsync(null, CommandLineParseResult.Empty); return (AggregatedConfiguration)configuration; } private static IReadOnlyDictionary OptionsByName(params CommandLineOption[] options) { var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (CommandLineOption option in options) { dict[option.Name] = option; } return dict; } private sealed class TestProvider : ICommandLineOptionsProvider { private readonly CommandLineOption[] _options; private readonly Func>? _validateOptionArgumentsAsync; public TestProvider( params CommandLineOption[] options) : this(validateOptionArgumentsAsync: null, options) { } public TestProvider( CommandLineOption option, Func> validateOptionArgumentsAsync) : this(validateOptionArgumentsAsync, option) { } private TestProvider( Func>? validateOptionArgumentsAsync, params CommandLineOption[] options) { _options = options; _validateOptionArgumentsAsync = validateOptionArgumentsAsync; } public string Uid => nameof(TestProvider); public string Version => "1.0.0"; public string DisplayName => nameof(TestProvider); public string Description => "Test provider"; public Task IsEnabledAsync() => Task.FromResult(true); public IReadOnlyCollection GetCommandLineOptions() => _options; public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) => _validateOptionArgumentsAsync?.Invoke(commandOption, arguments) ?? Task.FromResult(ValidationResult.Valid()); public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) => Task.FromResult(ValidationResult.Valid()); } }