diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj index 5d44f04283..9d6f20686e 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj @@ -51,6 +51,11 @@ $(DefineConstants);WINDOWS_UWP + + + $(DefineConstants);WIN_UI + + diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs index f15fd04dda..4fb87cc30e 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs @@ -9,8 +9,10 @@ using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -41,8 +43,7 @@ protected override Task SynchronizedDiscoverTestsAsync(VSTestDiscoverTestExecuti Debugger.Launch(); } - new MSTestDiscoverer().DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true); - return Task.CompletedTask; + return new MSTestDiscoverer(new TestSourceHandler(), CreateTelemetrySender()).DiscoverTestsAsync(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true); } /// @@ -55,7 +56,7 @@ protected override async Task SynchronizedRunTestsAsync(VSTestRunTestExecutionRe Debugger.Launch(); } - MSTestExecutor testExecutor = new(cancellationToken); + MSTestExecutor testExecutor = new(cancellationToken, CreateTelemetrySender()); await testExecutor.RunTestsAsync(request.AssemblyPaths, request.RunContext, request.FrameworkHandle, _configuration, isMTP: true).ConfigureAwait(false); } @@ -103,5 +104,19 @@ private static TestMethodIdentifierProperty GetMethodIdentifierPropertyFromManag // Or alternatively, does VSTest object model expose the assembly full name somewhere? return new TestMethodIdentifierProperty(assemblyFullName: string.Empty, @namespace, typeName, methodName, arity, parameterTypes, returnTypeFullName: string.Empty); } + + [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")] + private Func, Task>? CreateTelemetrySender() + { + ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation(); + if (!telemetryInformation.IsEnabled) + { + return null; + } + + ITelemetryCollector telemetryCollector = ServiceProvider.GetTelemetryCollector(); + + return (eventName, metrics) => telemetryCollector.LogEventAsync(eventName, metrics, CancellationToken.None); + } } #endif diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 5b46c15750..a98e00c83a 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -20,14 +20,27 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestDiscoverer : ITestDiscoverer { private readonly ITestSourceHandler _testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif + // The parameterless constructor is required by VSTest, which instantiates the + // discoverer via reflection. The internal constructor exists for tests and for the + // MTP bridge (MSTestBridgedTestFramework) which injects a telemetry sender. public MSTestDiscoverer() : this(new TestSourceHandler()) { } - internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler) - => _testSourceHandler = testSourceHandler; + internal MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) + { + _testSourceHandler = testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif + } /// /// Discovers the tests available from the provided source. Not supported for .xap source. @@ -39,9 +52,12 @@ public MSTestDiscoverer() [System.Security.SecurityCritical] [SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "0", Justification = "Discovery context can be null.")] public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) - => DiscoverTests(sources, discoveryContext, logger, discoverySink, null, isMTP: false); + // VSTest's ITestDiscoverer is a synchronous interface. The telemetry sender is null in + // this code path (only the MTP bridge supplies one), so the awaited send below completes + // synchronously and GetAwaiter().GetResult() does not actually block on I/O. + => DiscoverTestsAsync(sources, discoveryContext, logger, discoverySink, configuration: null, isMTP: false).GetAwaiter().GetResult(); - internal void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IConfiguration? configuration, bool isMTP) + internal async Task DiscoverTestsAsync(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IConfiguration? configuration, bool isMTP) { if (sources is null) { @@ -58,9 +74,31 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco throw new ArgumentNullException(nameof(discoverySink)); } - if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + // Initialize telemetry collection if not already set (e.g. first call in the session). +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + _ = MSTestTelemetryDataCollector.EnsureInitialized(); + } +#endif + + try + { + if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + { + new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP); + } + } + finally { - new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP); +#if !WINDOWS_UWP && !WIN_UI + // Send the discovery telemetry event ('mstest/discovery'). This always runs at the + // end of discovery — for discover-only sessions it is the only event; for sessions + // where a run follows, MSTestExecutor will send a separate 'mstest/sessionexit' event + // carrying assertion usage. Keeping the two events distinct avoids settings/attribute + // duplication and lets each event be self-contained. + await MSTestTelemetryDataCollector.SendDiscoveryTelemetryAndResetAsync(_telemetrySender).ConfigureAwait(false); +#endif } } } diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 098ac21a30..4fea4e6755 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -20,6 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestExecutor : ITestExecutor { private readonly CancellationToken _cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif /// /// Token for canceling the test run. @@ -35,10 +38,15 @@ public MSTestExecutor() _cancellationToken = CancellationToken.None; } - internal MSTestExecutor(CancellationToken cancellationToken) + internal MSTestExecutor(CancellationToken cancellationToken, Func, Task>? telemetrySender = null) { TestExecutionManager = new TestExecutionManager(); _cancellationToken = cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif } /// @@ -115,12 +123,27 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotEmpty(tests); - if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - return; + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif + + try + { + if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) + { + return; + } - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle, IConfiguration? configuration, bool isMTP) @@ -143,14 +166,29 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotEmpty(sources); - TestSourceHandler testSourceHandler = new(); - if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - return; + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif + + try + { + TestSourceHandler testSourceHandler = new(); + if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) + { + return; + } - sources = testSourceHandler.GetTestSources(sources); - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + sources = testSourceHandler.GetTestSources(sources); + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } /// @@ -159,6 +197,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run public void Cancel() => _testRunCancellationToken?.Cancel(); +#if !WINDOWS_UWP && !WIN_UI + private Task SendTelemetryAsync() + => MSTestTelemetryDataCollector.SendExecutionTelemetryAndResetAsync(_telemetrySender); +#else + private static Task SendTelemetryAsync() + => Task.CompletedTask; +#endif + private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func runTestsAction) { ApartmentState? requestedApartmentState = MSTestSettings.RunConfigurationSettings.ExecutionApartmentState; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs index 238640f70a..7c55d7c514 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs @@ -49,6 +49,17 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec return null; } + // Track class-level attributes for telemetry (read Current per call so a session reset + // between TypeEnumerator construction and use cannot cause writes to land on an + // orphaned collector). +#if !WINDOWS_UWP && !WIN_UI + if (Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current is { } telemetryDataCollector) + { + Attribute[] classAttributes = _reflectHelper.GetCustomAttributesCached(_type); + telemetryDataCollector.TrackDiscoveredClass(classAttributes); + } +#endif + // If test class is valid, then get the tests return GetTests(warnings); } @@ -145,6 +156,9 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables }; Attribute[] attributes = _reflectHelper.GetCustomAttributesCached(method); +#if !WINDOWS_UWP && !WIN_UI + Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current?.TrackDiscoveredMethod(attributes); +#endif TestMethodAttribute? testMethodAttribute = null; List? workItemIds = null; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs index f200893c1b..01aa0e45e8 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs @@ -59,6 +59,18 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger CurrentSettings = settings; RunConfigurationSettings = runConfigurationSettings; + + // Track configuration source for telemetry. +#if !WINDOWS_UWP && !WIN_UI + if (MSTestTelemetryDataCollector.Current is { } telemetry) + { + telemetry.ConfigurationSource = configuration?["mstest"] is not null + ? "testconfig.json" + : !StringEx.IsNullOrEmpty(context?.RunSettings?.SettingsXml) + ? "runsettings" + : "none"; + } +#endif } private static void SetGlobalSettings(string runsettingsXml, MSTestSettings settings, IMessageLogger? logger) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs new file mode 100644 index 0000000000..dadc8a96db --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -0,0 +1,480 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !WINDOWS_UWP && !WIN_UI +using System.Security.Cryptography; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + +/// +/// Collects and aggregates telemetry data about MSTest usage within a test session. +/// Captures settings, attribute usage, custom/inherited types, and assertion API usage. +/// +/// +/// This collector relies on static state () that lives in the +/// AppDomain in which the adapter executes. On .NET Framework runs that opt into the +/// adapter's child-AppDomain isolation, code that runs inside the child AppDomain (for +/// example, attribute discovery via the adapter's enumerators) sees its own +/// snapshot, which is initially null. In that case telemetry from +/// the isolated AppDomain is silently dropped (the Current?.Track* call sites are +/// null-safe). This is an intentional, graceful degradation: the .NET Framework AppDomain +/// scenario is rare and the effort to marshal counters across AppDomain boundaries via +/// is not justified for best-effort usage telemetry. +/// +internal sealed class MSTestTelemetryDataCollector +{ + private readonly Dictionary _attributeCounts = []; + private readonly HashSet _customTestMethodTypes = []; + private readonly HashSet _customTestClassTypes = []; + +#pragma warning disable IDE0032 // Use auto property - Volatile.Read/Write requires a ref to a field + private static MSTestTelemetryDataCollector? s_current; + private static int s_discoveryEventEmitted; +#pragma warning restore IDE0032 // Use auto property + + /// + /// Gets or sets the current telemetry data collector for the session. + /// Set at session start, cleared at session close. + /// + internal static MSTestTelemetryDataCollector? Current + { + get => Volatile.Read(ref s_current); + set => Volatile.Write(ref s_current, value); + } + + internal static MSTestTelemetryDataCollector EnsureInitialized() + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not null) + { + return collector; + } + + collector = new MSTestTelemetryDataCollector(); + MSTestTelemetryDataCollector? existingCollector = Interlocked.CompareExchange(ref s_current, collector, null); + + return existingCollector ?? collector; + } + + /// + /// Checks whether telemetry collection is opted out via environment variables. + /// Mirrors the same checks as Microsoft.Testing.Platform's TelemetryManager. + /// + /// true if telemetry is opted out; false otherwise. + internal static bool IsTelemetryOptedOut() + { + string? telemetryOptOut = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + if (telemetryOptOut is "1" or "true") + { + return true; + } + + string? cliTelemetryOptOut = Environment.GetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT"); + + return cliTelemetryOptOut is "1" or "true"; + } + + /// + /// Gets or sets the configuration source used for this session. + /// + internal string? ConfigurationSource { get; set; } + + /// + /// Records the attributes found on a test method during discovery. + /// + /// The cached attributes from the method. + internal void TrackDiscoveredMethod(Attribute[] attributes) + { + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + string attributeName = attributeType.Name; + + // Track custom/inherited TestMethodAttribute types (store anonymized hash) + if (attribute is TestMethodAttribute && attributeType != typeof(TestMethodAttribute)) + { + _customTestMethodTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track attribute usage counts by base type name (only known MSTest attributes) + string? trackingName = attribute switch + { + TestMethodAttribute => nameof(TestMethodAttribute), + TestClassAttribute => nameof(TestClassAttribute), + DataRowAttribute => nameof(DataRowAttribute), + DynamicDataAttribute => nameof(DynamicDataAttribute), + TimeoutAttribute => nameof(TimeoutAttribute), + IgnoreAttribute => nameof(IgnoreAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + RetryBaseAttribute => nameof(RetryBaseAttribute), + ConditionBaseAttribute => nameof(ConditionBaseAttribute), + TestCategoryAttribute => nameof(TestCategoryAttribute), +#if !WIN_UI + DeploymentItemAttribute => nameof(DeploymentItemAttribute), +#endif + _ => null, + }; + + if (trackingName is not null) + { + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + } + + /// + /// Records the attributes found on a test class during discovery. + /// + /// The cached attributes from the class. + internal void TrackDiscoveredClass(Attribute[] attributes) + { + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeType.Name)); + } + + string? trackingName = attribute switch + { + TestClassAttribute => nameof(TestClassAttribute), + ParallelizeAttribute => nameof(ParallelizeAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + _ => null, + }; + + if (trackingName is not null) + { + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + } + + /// + /// Builds the discovery telemetry metrics dictionary (settings + discovery-time data). + /// Sent at the end of MTP discover-only sessions (e.g. dotnet test --list-tests). + /// + /// A dictionary of telemetry key-value pairs for the discovery event. + internal Dictionary BuildDiscoveryMetrics() + { + Dictionary metrics = []; + AddDiscoveryMetrics(metrics); + return metrics; + } + + /// + /// Builds the execution telemetry metrics dictionary. Sent at the end of an MSTest run + /// (MTP run mode or VSTest run mode). Always carries assertion usage. Also carries the + /// settings/attribute/config payload UNLESS a discovery event has already been emitted + /// during this process — that avoids duplicating the discovery payload across two events + /// when a host (such as a future MTP host) chooses to call both discover and run within + /// the same session. + /// + /// Drained assertion call counts captured during execution. + /// When true, also include the discovery metrics + /// (settings, config_source, attribute_usage, custom_test_*_types). False when the discovery + /// event already shipped these in this process. + /// A dictionary of telemetry key-value pairs for the sessionexit event. + internal Dictionary BuildExecutionMetrics(Dictionary assertionCounts, bool includeDiscoveryPayload) + { + Dictionary metrics = []; + + if (includeDiscoveryPayload) + { + AddDiscoveryMetrics(metrics); + } + + if (assertionCounts.Count > 0) + { + metrics["mstest.assertion_usage"] = SerializeDictionary(assertionCounts); + } + + return metrics; + } + + private void AddDiscoveryMetrics(Dictionary metrics) + { + // Settings + AddSettingsMetrics(metrics); + + // Configuration source (runsettings, testconfig.json, or none) + if (ConfigurationSource is not null) + { + metrics["mstest.config_source"] = ConfigurationSource; + } + + // Attribute usage (aggregated counts as JSON; serializer enforces ordinal sort) + if (_attributeCounts.Count > 0) + { + metrics["mstest.attribute_usage"] = SerializeDictionary(_attributeCounts); + } + + // Custom/inherited types (anonymized names; serializer enforces ordinal sort) + if (_customTestMethodTypes.Count > 0) + { + metrics["mstest.custom_test_method_types"] = SerializeCollection(_customTestMethodTypes); + } + + if (_customTestClassTypes.Count > 0) + { + metrics["mstest.custom_test_class_types"] = SerializeCollection(_customTestClassTypes); + } + } + + private static string SerializeCollection(IEnumerable values) + { + StringBuilder builder = new("["); + bool isFirst = true; + + foreach (string value in values.OrderBy(static x => x, StringComparer.Ordinal)) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value); + isFirst = false; + } + + builder.Append(']'); + return builder.ToString(); + } + + private static string SerializeDictionary(IEnumerable> values) + { + StringBuilder builder = new("{"); + bool isFirst = true; + + foreach (KeyValuePair value in values.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value.Key); + builder.Append(':'); + builder.Append(value.Value.ToString(CultureInfo.InvariantCulture)); + isFirst = false; + } + + builder.Append('}'); + return builder.ToString(); + } + + private static void AppendJsonString(StringBuilder builder, string value) + { + builder.Append('"'); + + foreach (char character in value) + { + switch (character) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (char.IsControl(character)) + { + builder.Append("\\u"); + builder.Append(((int)character).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + builder.Append(character); + } + + break; + } + } + + builder.Append('"'); + } + + private static void AddSettingsMetrics(Dictionary metrics) + { + MSTestSettings settings = MSTestSettings.CurrentSettings; + + // Parallelization + metrics["mstest.setting.parallelization_enabled"] = AsTelemetryBool(!settings.DisableParallelization); + if (settings.ParallelizationScope is not null) + { + metrics["mstest.setting.parallelization_scope"] = settings.ParallelizationScope.Value.ToString(); + } + + if (settings.ParallelizationWorkers is not null) + { + metrics["mstest.setting.parallelization_workers"] = settings.ParallelizationWorkers.Value; + } + + // Timeouts + metrics["mstest.setting.test_timeout"] = settings.TestTimeout; + metrics["mstest.setting.assembly_initialize_timeout"] = settings.AssemblyInitializeTimeout; + metrics["mstest.setting.assembly_cleanup_timeout"] = settings.AssemblyCleanupTimeout; + metrics["mstest.setting.class_initialize_timeout"] = settings.ClassInitializeTimeout; + metrics["mstest.setting.class_cleanup_timeout"] = settings.ClassCleanupTimeout; + metrics["mstest.setting.test_initialize_timeout"] = settings.TestInitializeTimeout; + metrics["mstest.setting.test_cleanup_timeout"] = settings.TestCleanupTimeout; + metrics["mstest.setting.cooperative_cancellation"] = AsTelemetryBool(settings.CooperativeCancellationTimeout); + + // Behavior + metrics["mstest.setting.map_inconclusive_to_failed"] = AsTelemetryBool(settings.MapInconclusiveToFailed); + metrics["mstest.setting.map_not_runnable_to_failed"] = AsTelemetryBool(settings.MapNotRunnableToFailed); + metrics["mstest.setting.treat_discovery_warnings_as_errors"] = AsTelemetryBool(settings.TreatDiscoveryWarningsAsErrors); + metrics["mstest.setting.consider_empty_data_source_as_inconclusive"] = AsTelemetryBool(settings.ConsiderEmptyDataSourceAsInconclusive); + metrics["mstest.setting.order_tests_by_name"] = AsTelemetryBool(settings.OrderTestsByNameInClass); + metrics["mstest.setting.capture_debug_traces"] = AsTelemetryBool(settings.CaptureDebugTraces); + } + + // MTP's telemetry providers (e.g. AppInsightsProvider) reject raw boolean values and assert + // that they should be sent as their lowercase string form. This mirrors + // Microsoft.Testing.Platform.Telemetry.TelemetryExtensions.AsTelemetryBool, which we can't + // reference from here because that type lives in a different assembly. + private static string AsTelemetryBool(bool value) => value ? "true" : "false"; + + private static string AnonymizeString(string value) + { +#if NET + byte[] hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash); +#else + using var sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(value)); + return BitConverter.ToString(hash).Replace("-", string.Empty); +#endif + } + + /// + /// Sends the accumulated discovery telemetry via the provided sender delegate and clears the + /// discovery state by resetting the current collector. Safe to call when no sender is available + /// (the collector is still cleared so state does not leak across sessions). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendDiscoveryTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is null || telemetrySender is null) + { + return; + } + + // Defense in depth: re-check opt-out at send time in case the env var was set after + // EnsureInitialized but before this point. + if (IsTelemetryOptedOut()) + { + return; + } + + Dictionary metrics = collector.BuildDiscoveryMetrics(); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/discovery", metrics).ConfigureAwait(false); + + // Mark that the discovery payload (settings + attribute_usage + custom_test_*_types + // + config_source) has shipped in this process so a subsequent execution event in + // the same session does not duplicate it. + Interlocked.Exchange(ref s_discoveryEventEmitted, 1); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + // Clear the current collector so a subsequent execution accumulates settings/config + // anew (settings are static-per-process so re-population is cheap and keeps each + // event self-contained). + Current = null; + } + } + + /// + /// Sends the accumulated execution telemetry via the provided sender delegate, drains the + /// static assertion counters, and clears the current collector. Safe to call when no sender + /// is available (the counters and collector are still drained/cleared so state does not leak + /// across sessions). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendExecutionTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + // Always drain the static assertion counters so they don't leak across sessions, + // even when no sender is wired (VSTest mode) or no collector was initialized + // (e.g. telemetry opted out before EnsureInitialized was called). + Dictionary assertionCounts = TelemetryCollector.DrainAssertionCallCounts(); + + MSTestTelemetryDataCollector? collector = Current; + if (collector is null || telemetrySender is null) + { + return; + } + + // Defense in depth: re-check opt-out at send time in case the env var was set after + // EnsureInitialized but before this point. + if (IsTelemetryOptedOut()) + { + return; + } + + // If the discovery event already shipped the settings/attribute payload during this + // process, do not duplicate it in the sessionexit event. The flag is reset below in + // the finally block so each process can still ship a fresh payload after a full + // session reset. + bool includeDiscoveryPayload = Interlocked.CompareExchange(ref s_discoveryEventEmitted, 0, 0) == 0; + + Dictionary metrics = collector.BuildExecutionMetrics(assertionCounts, includeDiscoveryPayload); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics).ConfigureAwait(false); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + Current = null; + Interlocked.Exchange(ref s_discoveryEventEmitted, 0); + } + } +} +#endif diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs index 2b94ff37b7..4724dc2c02 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs @@ -63,7 +63,17 @@ internal sealed partial class AppInsightsProvider : TelemetryProperties.HostProperties.RuntimeIdentifierPropertyName, TelemetryProperties.HostProperties.ApplicationModePropertyName, TelemetryProperties.HostProperties.ExitCodePropertyName, - TelemetryProperties.HostProperties.ExtensionsPropertyName + TelemetryProperties.HostProperties.ExtensionsPropertyName, + + // MSTest session telemetry (see MSTestTelemetryDataCollector). These carry aggregated + // counts, anonymized SHA256 type hashes inside JSON envelopes, and well-known enum/string + // values - none of them contain unhashed user-identifying data. + "mstest.config_source", + "mstest.attribute_usage", + "mstest.custom_test_method_types", + "mstest.custom_test_class_types", + "mstest.assertion_usage", + "mstest.setting.parallelization_scope", ]; #endif diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 2beb00ee4a..80cb92adb2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -59,6 +59,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.InterpolatedStringHandlers.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.InterpolatedStringHandlers.cs index 49db337dde..1514f65efe 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.InterpolatedStringHandlers.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.InterpolatedStringHandlers.cs @@ -48,6 +48,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); ReportAssertAreEqualFailed(_expected, _actual, _builder.ToString()); } @@ -115,6 +116,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); ReportAssertAreNotEqualFailed(_notExpected, _actual, _builder.ToString()); } @@ -221,6 +223,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_failAction is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); _builder!.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); _failAction.Invoke(_builder!.ToString()); } @@ -327,6 +330,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_failAction is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); _builder!.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); _failAction.Invoke(_builder!.ToString()); } diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs index 95d917ae41..e25cb08b5a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs @@ -117,6 +117,8 @@ public static void AreEqual(float expected, float actual, float delta, [Interpol /// public static void AreEqual(float expected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -164,6 +166,8 @@ public static void AreNotEqual(float notExpected, float actual, float delta, [In /// public static void AreNotEqual(float notExpected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -232,6 +236,8 @@ public static void AreEqual(decimal expected, decimal actual, decimal delta, [In /// public static void AreEqual(decimal expected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -279,6 +285,8 @@ public static void AreNotEqual(decimal notExpected, decimal actual, decimal delt /// public static void AreNotEqual(decimal notExpected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -329,6 +337,8 @@ public static void AreEqual(long expected, long actual, long delta, [Interpolate /// public static void AreEqual(long expected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -376,6 +386,8 @@ public static void AreNotEqual(long notExpected, long actual, long delta, [Inter /// public static void AreNotEqual(long notExpected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -425,6 +437,8 @@ public static void AreEqual(double expected, double actual, double delta, [Inter /// public static void AreEqual(double expected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -472,6 +486,8 @@ public static void AreNotEqual(double notExpected, double actual, double delta, /// public static void AreNotEqual(double notExpected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs index aafc366e5c..3783ddb803 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs @@ -127,6 +127,8 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, /// public static void AreEqual(string? expected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + CheckParameterNotNull(culture, "Assert.AreEqual", nameof(culture)); if (!AreEqualFailing(expected, actual, ignoreCase, culture)) { @@ -223,6 +225,8 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC /// public static void AreNotEqual(string? notExpected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + CheckParameterNotNull(culture, "Assert.AreNotEqual", "culture"); if (!AreNotEqualFailing(notExpected, actual, ignoreCase, culture)) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs index 3142d9644f..f168d5d366 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs @@ -126,6 +126,8 @@ public static void AreEqual(T? expected, T? actual, IEqualityComparer? com /// public static void AreEqual(T? expected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (!AreEqualFailing(expected, actual, comparer)) { return; @@ -342,6 +344,8 @@ public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (!AreNotEqualFailing(notExpected, actual, comparer)) { return; diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs index 9b92297c9d..0b40cd8609 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string expectedExpression, string actualExpressio { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "expected", expectedExpression, "actual", actualExpression) + " "); ReportAssertAreSameFailed(_expected, _actual, _builder.ToString()); } @@ -98,6 +99,7 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionTwoParametersMessage, "notExpected", notExpectedExpression, "actual", actualExpression) + " "); ReportAssertAreNotSameFailed(_notExpected, _actual, _builder.ToString()); } @@ -176,6 +178,8 @@ public static void AreSame(T? expected, T? actual, [InterpolatedStringHandler /// public static void AreSame(T? expected, T? actual, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); + if (!IsAreSameFailing(expected, actual)) { return; @@ -250,6 +254,8 @@ public static void AreNotSame(T? notExpected, T? actual, [InterpolatedStringH /// public static void AreNotSame(T? notExpected, T? actual, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); + if (IsAreNotSameFailing(notExpected, actual)) { ReportAssertAreNotSameFailed(notExpected, actual, BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs index ecdbfa1407..d06dca873a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs @@ -154,6 +154,8 @@ public static T ContainsSingle(IEnumerable collection, string? message = " /// The item that matches the predicate. public static T ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + T firstMatch = default!; int matchCount = 0; @@ -209,6 +211,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// The item that matches the predicate. public static object? ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + object? firstMatch = null; int matchCount = 0; @@ -273,6 +277,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// public static void Contains(T expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -296,6 +302,8 @@ public static void Contains(T expected, IEnumerable collection, string? me /// public static void Contains(object? expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); foreach (object? item in collection) @@ -328,6 +336,8 @@ public static void Contains(object? expected, IEnumerable collection, string? me /// public static void Contains(T expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected, comparer)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -352,6 +362,8 @@ public static void Contains(T expected, IEnumerable collection, IEqualityC /// public static void Contains(object? expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(comparer, "Assert.Contains", "comparer"); @@ -384,6 +396,8 @@ public static void Contains(object? expected, IEnumerable collection, IEqualityC /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -407,6 +421,8 @@ public static void Contains(Func predicate, IEnumerable collectio /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(predicate, "Assert.Contains", "predicate"); @@ -486,6 +502,8 @@ public static void Contains(string substring, string value, string? message = "" /// public static void Contains(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(value, "Assert.Contains", "value"); CheckParameterNotNull(substring, "Assert.Contains", "substring"); @@ -522,6 +540,8 @@ public static void Contains(string substring, string value, StringComparison com /// public static void DoesNotContain(T notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -545,6 +565,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, s /// public static void DoesNotContain(object? notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); foreach (object? item in collection) @@ -575,6 +597,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, s /// public static void DoesNotContain(T notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected, comparer)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -599,6 +623,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, I /// public static void DoesNotContain(object? notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(comparer, "Assert.DoesNotContain", "comparer"); @@ -629,6 +655,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, I /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -652,6 +680,8 @@ public static void DoesNotContain(Func predicate, IEnumerable col /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(predicate, "Assert.DoesNotContain", "predicate"); @@ -729,6 +759,8 @@ public static void DoesNotContain(string substring, string value, string? messag /// public static void DoesNotContain(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(value, "Assert.DoesNotContain", "value"); CheckParameterNotNull(substring, "Assert.DoesNotContain", "substring"); @@ -772,6 +804,8 @@ public static void DoesNotContain(string substring, string value, StringComparis public static void IsInRange(T minValue, T maxValue, T value, string? message = "", [CallerArgumentExpression(nameof(minValue))] string minValueExpression = "", [CallerArgumentExpression(nameof(maxValue))] string maxValueExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsInRange"); + if (maxValue.CompareTo(minValue) < 0) { throw new ArgumentOutOfRangeException(nameof(maxValue), FrameworkMessages.IsInRangeMaxValueMustBeGreaterThanOrEqualMinValue); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs index 4ad00d8d02..1695475822 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs @@ -47,6 +47,7 @@ internal void ComputeAssertion(string assertionName, string collectionExpression { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertionName)); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "collection", collectionExpression) + " "); ReportAssertCountFailed(assertionName, _expectedCount, _actualCount, _builder.ToString()); } @@ -115,6 +116,7 @@ internal void ComputeAssertion(string collectionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "collection", collectionExpression) + " "); ReportAssertIsNotEmptyFailed(_builder.ToString()); } @@ -197,6 +199,8 @@ public static void IsNotEmpty(IEnumerable collection, [InterpolatedStringH /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Any()) { return; @@ -217,6 +221,8 @@ public static void IsNotEmpty(IEnumerable collection, string? message = "" /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Cast().Any()) { return; @@ -320,6 +326,8 @@ public static void IsEmpty(IEnumerable collection, string? message = "", [Caller private static void HasCount(string assertionName, int expected, IEnumerable collection, string? message, string collectionExpression) { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertionName)); + int actualCount = collection.Count(); if (actualCount == expected) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs index 9ebe0a817c..bdd47da320 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs @@ -72,6 +72,8 @@ public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? /// public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedSuffix))] string expectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.EndsWith"); + CheckParameterNotNull(value, "Assert.EndsWith", "value"); CheckParameterNotNull(expectedSuffix, "Assert.EndsWith", "expectedSuffix"); if (!value.EndsWith(expectedSuffix, comparisonType)) @@ -146,6 +148,8 @@ public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] /// public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedSuffix))] string notExpectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotEndWith"); + CheckParameterNotNull(value, "Assert.DoesNotEndWith", "value"); CheckParameterNotNull(notExpectedSuffix, "Assert.DoesNotEndWith", "notExpectedSuffix"); if (value.EndsWith(notExpectedSuffix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs index cbe187dc13..e377044d5a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs @@ -22,5 +22,8 @@ public sealed partial class Assert /// [DoesNotReturn] public static void Fail(string message = "") - => ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + { + TelemetryCollector.TrackAssertionCall("Assert.Fail"); + ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + } } diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs index ba7f415a54..0d1d634c01 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs @@ -44,6 +44,8 @@ public sealed partial class Assert public static void IsGreaterThan(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThan"); + if (value.CompareTo(lowerBound) > 0) { return; @@ -89,6 +91,8 @@ public static void IsGreaterThan(T lowerBound, T value, string? message = "", public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThanOrEqualTo"); + if (value.CompareTo(lowerBound) >= 0) { return; @@ -134,6 +138,8 @@ public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? mess public static void IsLessThan(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThan"); + if (value.CompareTo(upperBound) < 0) { return; @@ -179,6 +185,8 @@ public static void IsLessThan(T upperBound, T value, string? message = "", [C public static void IsLessThanOrEqualTo(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThanOrEqualTo"); + if (value.CompareTo(upperBound) <= 0) { return; @@ -216,6 +224,8 @@ public static void IsLessThanOrEqualTo(T upperBound, T value, string? message public static void IsPositive(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsPositive"); + var zero = default(T); // Handle special case for floating point NaN values @@ -270,6 +280,8 @@ public static void IsPositive(T value, string? message = "", [CallerArgumentE public static void IsNegative(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsNegative"); + var zero = default(T); // Handle special case for floating point NaN values diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs index 6923b71fbf..ecd743b0c9 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs @@ -23,6 +23,8 @@ public sealed partial class Assert [DoesNotReturn] public static void Inconclusive(string message = "") { + TelemetryCollector.TrackAssertionCall("Assert.Inconclusive"); + string userMessage = BuildUserMessage(message); throw new AssertInconclusiveException( FormatAssertionFailed("Assert.Inconclusive", userMessage)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs index 3404a7ac57..82019a3590 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsExactInstanceOfTypeFailed(_value, _expectedType, _builder.ToString()); } @@ -98,6 +99,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsExactInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -160,6 +162,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNotExactInstanceOfTypeFailed(_value, _wrongType, _builder.ToString()); } @@ -220,6 +223,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNotExactInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -289,6 +293,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); + if (IsExactInstanceOfTypeFailing(value, expectedType)) { ReportAssertIsExactInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -369,6 +375,8 @@ private static void ReportAssertIsExactInstanceOfTypeFailed(object? value, Type? /// public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + if (IsNotExactInstanceOfTypeFailing(value, wrongType)) { ReportAssertIsNotExactInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs index 27f06595b0..7e55cbeda6 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs @@ -38,6 +38,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsInstanceOfTypeFailed(_value, _expectedType, _builder.ToString()); } @@ -98,6 +99,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -160,6 +162,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNotInstanceOfTypeFailed(_value, _wrongType, _builder.ToString()); } @@ -220,6 +223,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNotInstanceOfTypeFailed(_value, typeof(TArg), _builder.ToString()); } @@ -290,6 +294,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); + if (IsInstanceOfTypeFailing(value, expectedType)) { ReportAssertIsInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -372,6 +378,8 @@ private static void ReportAssertIsInstanceOfTypeFailed(object? value, Type? expe /// public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + if (IsNotInstanceOfTypeFailing(value, wrongType)) { ReportAssertIsNotInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs index 00b3e1eec6..7c6057ce93 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs @@ -36,6 +36,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNullFailed(_builder.ToString()); } @@ -90,6 +91,7 @@ internal void ComputeAssertion(string valueExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "value", valueExpression) + " "); ReportAssertIsNotNullFailed(_builder.ToString()); } @@ -150,6 +152,8 @@ public static void IsNull(object? value, [InterpolatedStringHandlerArgument(name /// public static void IsNull(object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); + if (IsNullFailing(value)) { ReportAssertIsNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); @@ -189,6 +193,8 @@ public static void IsNotNull([NotNull] object? value, [InterpolatedStringHandler /// public static void IsNotNull([NotNull] object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); + if (IsNotNullFailing(value)) { ReportAssertIsNotNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs index d1480fe701..f88fda5e27 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs @@ -36,6 +36,7 @@ internal void ComputeAssertion(string conditionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "condition", conditionExpression) + " "); ReportAssertIsTrueFailed(_builder.ToString()); } @@ -88,6 +89,7 @@ internal void ComputeAssertion(string conditionExpression) { if (_builder is not null) { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); _builder.Insert(0, string.Format(CultureInfo.CurrentCulture, FrameworkMessages.CallerArgumentExpressionSingleParameterMessage, "condition", conditionExpression) + " "); ReportAssertIsFalseFailed(_builder.ToString()); } @@ -148,6 +150,8 @@ public static void IsTrue([DoesNotReturnIf(false)] bool? condition, [Interpolate /// public static void IsTrue([DoesNotReturnIf(false)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); + if (IsTrueFailing(condition)) { ReportAssertIsTrueFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); @@ -186,6 +190,8 @@ public static void IsFalse([DoesNotReturnIf(true)] bool? condition, [Interpolate /// public static void IsFalse([DoesNotReturnIf(true)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); + if (IsFalseFailing(condition)) { ReportAssertIsFalseFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs index 23b69859c7..c36ff1ef52 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs @@ -39,6 +39,8 @@ public sealed partial class Assert /// public static void MatchesRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.MatchesRegex"); + CheckParameterNotNull(value, "Assert.MatchesRegex", "value"); CheckParameterNotNull(pattern, "Assert.MatchesRegex", "pattern"); @@ -115,6 +117,8 @@ public static void MatchesRegex([NotNull] string? pattern, [NotNull] string? val /// public static void DoesNotMatchRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotMatchRegex"); + CheckParameterNotNull(value, "Assert.DoesNotMatchRegex", "value"); CheckParameterNotNull(pattern, "Assert.DoesNotMatchRegex", "pattern"); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs index caf58c2a10..c478b3cb44 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs @@ -72,6 +72,8 @@ public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string /// public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedPrefix))] string expectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.StartsWith"); + CheckParameterNotNull(value, "Assert.StartsWith", "value"); CheckParameterNotNull(expectedPrefix, "Assert.StartsWith", "expectedPrefix"); if (!value.StartsWith(expectedPrefix, comparisonType)) @@ -144,6 +146,8 @@ public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNul /// public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedPrefix))] string notExpectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotStartWith"); + CheckParameterNotNull(value, "Assert.DoesNotStartWith", "value"); CheckParameterNotNull(notExpectedPrefix, "Assert.DoesNotStartWith", "notExpectedPrefix"); if (value.StartsWith(notExpectedPrefix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.That.cs b/src/TestFramework/TestFramework/Assertions/Assert.That.cs index 0894543cc7..382a53f397 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.That.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.That.cs @@ -26,6 +26,8 @@ public static partial class AssertExtensions /// Thrown if the evaluated condition is . public static void That(Expression> condition, string? message = null, [CallerArgumentExpression(nameof(condition))] string? conditionExpression = null) { + TelemetryCollector.TrackAssertionCall("Assert.That"); + if (condition == null) { throw new ArgumentNullException(nameof(condition)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs index 5c39d4138e..6ba7c69851 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs @@ -321,6 +321,8 @@ public static TException ThrowsExactly(Func action, [Interp private static TException ThrowsException(Action action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = message ?? throw new ArgumentNullException(nameof(message)); @@ -341,6 +343,8 @@ private static TException ThrowsException(Action action, bool isStri private static TException ThrowsException(Action action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = messageBuilder ?? throw new ArgumentNullException(nameof(messageBuilder)); @@ -477,6 +481,8 @@ public static Task ThrowsExactlyAsync(Func action, private static async Task ThrowsExceptionAsync(Func action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = message ?? throw new ArgumentNullException(nameof(message)); @@ -497,6 +503,8 @@ private static async Task ThrowsExceptionAsync(Func ThrowsExceptionAsync(Func action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = messageBuilder ?? throw new ArgumentNullException(nameof(messageBuilder)); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs index 3ace0a74dc..bccbe1cf9a 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs @@ -58,6 +58,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual) /// public static void AreEqual(ICollection? expected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, new ObjectComparer(), ref reason)) { @@ -112,6 +114,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual) /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, new ObjectComparer(), ref reason)) { @@ -170,6 +174,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull /// public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, comparer, ref reason)) { @@ -228,6 +234,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [N /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, comparer, ref reason)) { diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs index 022604dd90..a4218b4b86 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs @@ -118,6 +118,8 @@ public static void AreEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? expected, [NotNullIfNotNull(nameof(expected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -284,6 +286,8 @@ public static void AreNotEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? notExpected, [NotNullIfNotNull(nameof(notExpected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs index 189ffadae1..5203a843eb 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs @@ -50,6 +50,8 @@ public static void Contains([NotNull] ICollection? collection, object? element) /// public static void Contains([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.Contains"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.Contains", "collection"); foreach (object? current in collection) @@ -101,6 +103,8 @@ public static void DoesNotContain([NotNull] ICollection? collection, object? ele /// public static void DoesNotContain([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.DoesNotContain"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.DoesNotContain", "collection"); foreach (object? current in collection) @@ -141,6 +145,8 @@ public static void AllItemsAreNotNull([NotNull] ICollection? collection) /// public static void AllItemsAreNotNull([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreNotNull"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreNotNull", "collection"); foreach (object? current in collection) { @@ -183,6 +189,8 @@ public static void AllItemsAreUnique([NotNull] ICollection? collection) /// public static void AllItemsAreUnique([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreUnique"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreUnique", "collection"); message = Assert.ReplaceNulls(message); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs index 7d5fff305b..4c9f2f9d19 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs @@ -54,6 +54,8 @@ public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollecti /// public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -114,6 +116,8 @@ public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] IColle /// public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsNotSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsNotSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsNotSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs index eeae04b966..72dc9fd796 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs @@ -57,6 +57,8 @@ public static void AllItemsAreInstancesOfType([NotNull] ICollection? collection, public static void AllItemsAreInstancesOfType( [NotNull] ICollection? collection, [NotNull] Type? expectedType, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreInstancesOfType"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreInstancesOfType", "collection"); Assert.CheckParameterNotNull(expectedType, "CollectionAssert.AllItemsAreInstancesOfType", "expectedType"); int i = 0; diff --git a/src/TestFramework/TestFramework/Assertions/StringAssert.cs b/src/TestFramework/TestFramework/Assertions/StringAssert.cs index 3c65a1ed62..7721f51cd4 100644 --- a/src/TestFramework/TestFramework/Assertions/StringAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/StringAssert.cs @@ -116,6 +116,8 @@ public static void Contains([NotNull] string? value, [NotNull] string? substring /// public static void Contains([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Contains"); + Assert.CheckParameterNotNull(value, "StringAssert.Contains", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.Contains", "substring"); if (value.IndexOf(substring, comparisonType) < 0) @@ -213,6 +215,8 @@ public static void StartsWith([NotNull] string? value, [NotNull] string? substri /// public static void StartsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.StartsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.StartsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.StartsWith", "substring"); if (!value.StartsWith(substring, comparisonType)) @@ -310,6 +314,8 @@ public static void EndsWith([NotNull] string? value, [NotNull] string? substring /// public static void EndsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.EndsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.EndsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.EndsWith", "substring"); if (!value.EndsWith(substring, comparisonType)) @@ -364,6 +370,8 @@ public static void Matches([NotNull] string? value, [NotNull] Regex? pattern) /// public static void Matches([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Matches"); + Assert.CheckParameterNotNull(value, "StringAssert.Matches", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.Matches", "pattern"); @@ -415,6 +423,8 @@ public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? patter /// public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.DoesNotMatch"); + Assert.CheckParameterNotNull(value, "StringAssert.DoesNotMatch", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.DoesNotMatch", "pattern"); diff --git a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs new file mode 100644 index 0000000000..209a90e804 --- /dev/null +++ b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Collects aggregated telemetry data about MSTest API usage within a test session. +/// This data is used to understand which APIs are heavily used or unused to guide future investment. +/// +internal static class TelemetryCollector +{ + private static ConcurrentDictionary s_assertionCallCounts = new(); + + /// + /// Records that an assertion method was called. + /// + /// The full name of the assertion (e.g. "Assert.AreEqual", "CollectionAssert.Contains"). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void TrackAssertionCall(string assertionName) + => s_assertionCallCounts.AddOrUpdate(assertionName, 1, static (_, count) => count + 1); + + /// + /// Gets a snapshot of all assertion call counts and resets the counters. + /// This is thread-safe but best-effort: it atomically swaps the dictionary and copies the old one. + /// In-flight calls to that race with the swap may be lost. + /// This is acceptable for telemetry where approximate counts are sufficient. + /// + /// A dictionary mapping assertion names to their (best-effort) call counts. + internal static Dictionary DrainAssertionCallCounts() + { + ConcurrentDictionary old = Interlocked.Exchange(ref s_assertionCallCounts, new ConcurrentDictionary()); +#pragma warning disable IDE0028 // Simplify collection initialization - ConcurrentDictionary snapshot copy + return new Dictionary(old); +#pragma warning restore IDE0028 // Simplify collection initialization + } +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs new file mode 100644 index 0000000000..b09a2e72cb --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class TelemetryTests : AcceptanceTestBase +{ + private const string MTPAssetName = "TelemetryMTPProject"; + private const string TestResultsFolderName = "TestResults"; + + #region MTP mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_RunTests_SendsTelemetryWithSettingsAndAttributes(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + // In MTP run mode, MTP only invokes the executor (no separate discovery call), so the + // single sessionexit event carries the full picture: settings, config_source, attribute + // usage, and assertion usage. + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit +.+mstest\.setting\.parallelization_enabled +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.attribute_usage"), $"Expected attribute_usage in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.config_source"), $"Expected config_source in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.assertion_usage"), $"Expected assertion_usage in telemetry.\n{content}"); + + // Regression guard: discovery + execution data must ship in a single sessionexit event, + // not split across two (an earlier iteration of this code did the latter). + MatchCollection sessionExitEvents = Regex.Matches(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"); + Assert.HasCount(1, sessionExitEvents, $"Expected exactly one MSTest sessionexit telemetry event, found {sessionExitEvents.Count}.\n{content}"); + } + + #endregion + + #region MTP mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_DiscoverTests_SendsTelemetryEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--list-tests --diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/discovery[\s\S]+?mstest\.attribute_usage +"""; + await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + } + + #endregion + + #region MTP mode - Telemetry disabled + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_WhenTelemetryDisabled_DoesNotSendMSTestEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + new Dictionary + { + { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" }, + }, + disableTelemetry: false, + TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string diagContentsPattern = +""" +.+ Microsoft.Testing.Platform.Telemetry.TelemetryManager DEBUG Telemetry is 'DISABLED' +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"), + "MSTest sessionexit telemetry event should not be sent when telemetry is disabled."); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/discovery"), + "MSTest discovery telemetry event should not be sent when telemetry is disabled."); + } + + #endregion + + #region VSTest mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_RunTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --framework {tfm}", + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("Passed!"); + } + + #endregion + + #region VSTest mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_DiscoverTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --framework {tfm} --list-tests", + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test --list-tests failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("PassingTest"); + testResult.AssertOutputContains("DataDrivenTest"); + testResult.AssertOutputContains("TestWithTimeout"); + } + + #endregion + + #region Helpers + + private static async Task AssertDiagnosticReportAsync(TestHostResult testHostResult, string diagPathPattern, string diagContentsPattern, string level = "Trace", string flushType = "async") + { + string outputPattern = $""" +Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + Match match = Regex.Match(testHostResult.StandardOutput, diagPathPattern); + Assert.IsTrue(match.Success, $"{testHostResult}\n{diagPathPattern}"); + + (bool success, string content) = await CheckDiagnosticContentsMatchAsync(match.Value, diagContentsPattern); + Assert.IsTrue(success, $"{content}\n{diagContentsPattern}"); + + return match.Value; + } + + private static async Task<(bool IsMatch, string Content)> CheckDiagnosticContentsMatchAsync(string path, string pattern) + { + using var reader = new StreamReader(path); + string content = await reader.ReadToEndAsync(); + + return (Regex.IsMatch(content, pattern), content); + } + + #endregion + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string AssetId = nameof(TelemetryTests); + + public string MTPProjectPath => GetAssetPath(AssetId); + + public string VSTestProjectPath => Path.Combine(GetAssetPath(AssetId), "vstest"); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() + => (AssetId, MTPAssetName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion)); + + private const string SourceCode = """ +#file TelemetryMTPProject.csproj + + + + Exe + true + $TargetFrameworks$ + Preview + + + + + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} + +#file vstest/TelemetryVSTestProject.csproj + + + + $TargetFrameworks$ + Preview + false + true + + + + + + + + + + +#file vstest/global.json +{ + "test": { + "runner": "VSTest" + } +} + +#file vstest/UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + } + + public TestContext TestContext { get; set; } +}