Add Azure DevOps flaky-history annotations and quarantine awareness

Directory.Packages.props

@@ -59,6 +59,7 @@
     <PackageVersion Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageVersion Include="Microsoft.WindowsAppSDK" Version="1.8.251003001" />
     <PackageVersion Include="OpenTelemetry" Version="1.15.3" />
+    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
   </ItemGroup>
   <ItemGroup Label="Test dependencies">

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs

@@ -6,5 +6,8 @@ namespace Microsoft.Testing.Extensions.Reporting;
 internal static class AzureDevOpsCommandLineOptions
 {
     public const string AzureDevOpsOptionName = "report-azdo";
+    public const string AzureDevOpsDemoteKnownFlaky = "report-azdo-demote-known-flaky";
+    public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history";
+    public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file";
     public const string AzureDevOpsReportSeverity = "report-azdo-severity";
 }

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using Microsoft.Testing.Extensions.AzureDevOpsReport;
 using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources;
 using Microsoft.Testing.Platform.CommandLine;
 using Microsoft.Testing.Platform.Extensions;
@@ -12,6 +13,11 @@ internal sealed class AzureDevOpsCommandLineProvider : ICommandLineOptionsProvid
 {
     private static readonly string[] SeverityOptions = ["error", "warning"];
 
+    private static readonly string DemoteKnownFlakyOptionDescriptionFormatted = string.Format(
+        CultureInfo.InvariantCulture,
+        AzureDevOpsResources.DemoteKnownFlakyOptionDescription,
+        AzureDevOpsReporter.KnownFlakyFailureRateThreshold * 100);
+
     public string Uid => nameof(AzureDevOpsCommandLineProvider);
 
     public string Version => ExtensionVersion.DefaultSemVer;
@@ -26,31 +32,61 @@ public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
         =>
         [
             new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky, DemoteKnownFlakyOptionDescriptionFormatted, ArgumentArity.Zero, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
+            new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false),
             new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
         ];
 
     public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
+        => commandOption.Name switch
+        {
+            AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments),
+            AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity => ValidateSeverityArgumentsAsync(arguments),
+            _ => ValidationResult.ValidTask,
+        };
+
+    public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
     {
-        if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)
+        string? errorMessage = null;
+        if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName))
         {
-            if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase))
+            if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky))
             {
-                return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
+                errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsFlakyHistoryRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsQuarantineFileRequiresAzureDevOps;
+            }
+            else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
+            {
+                errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps;
             }
         }
-
-        return ValidationResult.ValidTask;
-    }
-
-    public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
-    {
-        if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) &&
-            commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
+        else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky)
+            && !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
         {
-            // If report-azdo is not set, but report-azdo-severity is set, it's invalid.
-            return ValidationResult.InvalidTask(AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps);
+            errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory;
         }
 
-        return ValidationResult.ValidTask;
+        return errorMessage is null
+            ? ValidationResult.ValidTask
+            : ValidationResult.InvalidTask(errorMessage);
     }
+
+    private static Task<ValidationResult> ValidateFlakyHistoryArgumentsAsync(string[] arguments)
+        => int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days)
+            && days is >= 1 and <= 90
+                ? ValidationResult.ValidTask
+                : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0]));
+
+    private static Task<ValidationResult> ValidateSeverityArgumentsAsync(string[] arguments)
+        => SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase)
+            ? ValidationResult.ValidTask
+            : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
 }

src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs

@@ -4,7 +4,6 @@
 using Microsoft.Testing.Extensions.AzureDevOpsReport;
 using Microsoft.Testing.Extensions.Reporting;
 using Microsoft.Testing.Platform.Builder;
-using Microsoft.Testing.Platform.Extensions;
 using Microsoft.Testing.Platform.Services;
 
 namespace Microsoft.Testing.Extensions;
@@ -20,17 +19,31 @@ public static class AzureDevOpsExtensions
     /// <param name="builder">The test application builder.</param>
     public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
     {
-        var compositeTestSessionAzDoService =
-           new CompositeExtensionFactory<AzureDevOpsReporter>(serviceProvider =>
-               new AzureDevOpsReporter(
-                   serviceProvider.GetCommandLineOptions(),
-                   serviceProvider.GetEnvironment(),
-                   serviceProvider.GetFileSystem(),
-                   serviceProvider.GetOutputDevice(),
-                   serviceProvider.GetLoggerFactory()));
+        AzureDevOpsHistoryService? historyService = null;
 
-        builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService);
+        builder.TestHost.AddDataConsumer(serviceProvider =>
+        {
+            historyService ??= CreateHistoryService(serviceProvider);
 
+            return new AzureDevOpsReporter(
+                serviceProvider.GetCommandLineOptions(),
+                serviceProvider.GetEnvironment(),
+                serviceProvider.GetFileSystem(),
+                serviceProvider.GetOutputDevice(),
+                serviceProvider.GetLoggerFactory(),
+                historyService);
+        });
+        builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider =>
+            historyService ??= CreateHistoryService(serviceProvider));
         builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
     }
+
+    private static AzureDevOpsHistoryService CreateHistoryService(IServiceProvider serviceProvider)
+        => new(
+            serviceProvider.GetCommandLineOptions(),
+            serviceProvider.GetEnvironment(),
+            serviceProvider.GetClock(),
+            new AzureDevOpsHistoryClient(serviceProvider.GetTask(), serviceProvider.GetClock()),
+            serviceProvider.GetTask(),
+            serviceProvider.GetLoggerFactory());
 }