From 87842459bba256bff7a54729b919079592c9ecc5 Mon Sep 17 00:00:00 2001
From: Shyam Namboodiripad <gnamboo@microsoft.com>
Date: Tue, 22 Jul 2025 19:54:06 -0700
Subject: [PATCH 1/2] Reference ApplicationInsights package in the dotnet
 aieval tool

---
 eng/packages/General.props                                     | 1 +
 .../Microsoft.Extensions.AI.Evaluation.Console.csproj          | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/eng/packages/General.props b/eng/packages/General.props
index 5478882d851..3dc7f81fc3d 100644
--- a/eng/packages/General.props
+++ b/eng/packages/General.props
@@ -8,6 +8,7 @@
     <PackageVersion Include="DnsClient" Version="1.8.0" />
     <PackageVersion Include="ICSharpCode.Decompiler" Version="9.1.0.7988" />
     <PackageVersion Include="Markdig.Signed" Version="$(MarkdigSignedVersion)" />
+    <PackageVersion Include="Microsoft.ApplicationInsights" Version="2.23.0" />
     <PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="$(MicrosoftCodeAnalysisAnalyzersVersion)" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Common" Version="$(MicrosoftCodeAnalysisVersion)" />
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
index e0dbc06d9df..cd01336f544 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
@@ -37,8 +37,9 @@
   <ItemGroup>
     <PackageReference Include="Azure.Identity" />
     <PackageReference Include="Azure.Storage.Files.DataLake" />
-    <PackageReference Include="System.CommandLine" />
+    <PackageReference Include="Microsoft.ApplicationInsights" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+    <PackageReference Include="System.CommandLine" />
   </ItemGroup>
 
   <ItemGroup>

From 6a88033a1cd22a6ee368c324bc7be842ddc53377 Mon Sep 17 00:00:00 2001
From: Shyam Namboodiripad <gnamboo@microsoft.com>
Date: Thu, 4 Sep 2025 17:35:27 -0700
Subject: [PATCH 2/2] Introduce telemetry in aieval dotnet tool

---
 .../Commands/CleanCacheCommand.cs             |  68 ++-
 .../Commands/CleanResultsCommand.cs           | 112 +++--
 .../Commands/ReportCommand.cs                 | 411 +++++++++++++++---
 ...ft.Extensions.AI.Evaluation.Console.csproj |   6 +-
 .../Program.cs                                |  47 +-
 .../TELEMETRY.md                              |  73 ++++
 .../Telemetry/DeviceIdHelper.cs               | 165 +++++++
 .../Telemetry/EnvironmentHelper.cs            |  65 +++
 .../Telemetry/TelemetryConstants.cs           | 174 ++++++++
 .../Telemetry/TelemetryExtensions.cs          | 228 ++++++++++
 .../Telemetry/TelemetryHelper.cs              | 184 ++++++++
 .../Utilities/LoggerExtensions.cs             |  26 +-
 .../BLEUEvaluator.cs                          |  28 +-
 .../F1Evaluator.cs                            |  18 +-
 .../GLEUEvaluator.cs                          |  21 +-
 ...rosoft.Extensions.AI.Evaluation.NLP.csproj |   2 +-
 ...ft.Extensions.AI.Evaluation.Quality.csproj |   2 +-
 .../ContentHarmEvaluator.cs                   |   2 +-
 .../ContentSafetyEvaluator.cs                 |   2 +-
 .../ContentSafetyService.cs                   |   2 +-
 ...oft.Extensions.AI.Evaluation.Safety.csproj |   2 +-
 .../EvaluationMetricExtensions.cs             |  41 +-
 .../EvaluationResultExtensions.cs             |  23 +-
 .../Microsoft.Extensions.AI.Evaluation.csproj |   4 +
 .../Microsoft.Extensions.AI.Evaluation.json   |   8 +
 ...Utilities.cs => BuiltInMetricUtilities.cs} |   9 +-
 .../Utilities/ModelInfo.cs                    |   2 +-
 .../Utilities/TimingHelper.cs                 |   6 +-
 .../NLPEvaluatorTests.cs                      |   3 -
 .../QualityEvaluatorTests.cs                  |  13 +-
 .../ResultsTests.cs                           |   2 -
 .../BLEUEvaluatorTests.cs                     |   6 +-
 .../F1EvaluatorTests.cs                       |   6 +-
 .../GLEUEvaluatorTests.cs                     |   6 +-
 .../SimpleTokenizerTests.cs                   |   2 -
 .../ResponseCacheTester.cs                    |   3 -
 .../ScenarioRunResultTests.cs                 |   2 +-
 .../SerializationChainingTests.cs             |  12 +-
 ...ests.cs => BuiltInMetricUtilitiesTests.cs} |  10 +-
 39 files changed, 1537 insertions(+), 259 deletions(-)
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/TELEMETRY.md
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs
 create mode 100644 src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs
 rename src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/{BuiltInEvaluatorUtilities.cs => BuiltInMetricUtilities.cs} (52%)
 rename test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/{BuiltInEvaluatorUtilitiesTests.cs => BuiltInMetricUtilitiesTests.cs} (78%)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs
index b0d975edb43..a3224e411cd 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs
@@ -2,46 +2,66 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Identity;
 using Azure.Storage.Files.DataLake;
+using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
 using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
 using Microsoft.Extensions.Logging;
+using static Microsoft.Extensions.AI.Evaluation.Console.Telemetry.TelemetryConstants;
 
 namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 
-internal sealed class CleanCacheCommand(ILogger logger)
+internal sealed class CleanCacheCommand(ILogger logger, TelemetryHelper telemetryHelper)
 {
-    internal async Task<int> InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default)
+    internal async Task<int> InvokeAsync(
+        DirectoryInfo? storageRootDir,
+        Uri? endpointUri,
+        CancellationToken cancellationToken = default)
     {
-        IEvaluationResponseCacheProvider cacheProvider;
-
-        if (storageRootDir is not null)
-        {
-            string storageRootPath = storageRootDir.FullName;
-            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
-            logger.LogInformation("Deleting expired cache entries...");
-
-            cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
-        }
-        else if (endpointUri is not null)
-        {
-            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
-
-            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
-            cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
-        }
-        else
-        {
-            throw new InvalidOperationException("Either --path or --endpoint must be specified");
-        }
+        var telemetryProperties = new Dictionary<string, string>();
 
         await logger.ExecuteWithCatchAsync(
-            () => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);
+            operation: () =>
+                telemetryHelper.ReportOperationAsync(
+                    operationName: EventNames.CleanCacheCommand,
+                    operation: () =>
+                    {
+                        IEvaluationResponseCacheProvider cacheProvider;
+
+                        if (storageRootDir is not null)
+                        {
+                            string storageRootPath = storageRootDir.FullName;
+                            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+                            logger.LogInformation("Deleting expired cache entries...");
+
+                            cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
+
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeDisk;
+                        }
+                        else if (endpointUri is not null)
+                        {
+                            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+
+                            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+                            cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
+
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeAzure;
+                        }
+                        else
+                        {
+                            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+                        }
+
+                        return cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken);
+                    },
+                    properties: telemetryProperties,
+                    logger: logger)).ConfigureAwait(false);
 
         return 0;
     }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs
index 8d6617d8302..ec5a8805424 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs
@@ -8,14 +8,16 @@
 using System.Threading.Tasks;
 using Azure.Identity;
 using Azure.Storage.Files.DataLake;
+using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
 using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
 using Microsoft.Extensions.Logging;
+using static Microsoft.Extensions.AI.Evaluation.Console.Telemetry.TelemetryConstants;
 
 namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 
-internal sealed class CleanResultsCommand(ILogger logger)
+internal sealed class CleanResultsCommand(ILogger logger, TelemetryHelper telemetryHelper)
 {
     internal async Task<int> InvokeAsync(
         DirectoryInfo? storageRootDir,
@@ -23,61 +25,81 @@ internal async Task<int> InvokeAsync(
         int lastN,
         CancellationToken cancellationToken = default)
     {
-        IEvaluationResultStore resultStore;
-
-        if (storageRootDir is not null)
-        {
-            string storageRootPath = storageRootDir.FullName;
-            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+        var telemetryProperties =
+            new Dictionary<string, string>
+            {
+                [PropertyNames.LastN] = lastN.ToTelemetryPropertyValue()
+            };
 
-            resultStore = new DiskBasedResultStore(storageRootPath);
-        }
-        else if (endpointUri is not null)
-        {
-            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+        await logger.ExecuteWithCatchAsync(
+            operation: () =>
+                telemetryHelper.ReportOperationAsync(
+                    operationName: EventNames.CleanResultsCommand,
+                    operation: async ValueTask () =>
+                    {
+                        IEvaluationResultStore resultStore;
 
-            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
-            resultStore = new AzureStorageResultStore(fsClient);
-        }
-        else
-        {
-            throw new InvalidOperationException("Either --path or --endpoint must be specified");
-        }
+                        if (storageRootDir is not null)
+                        {
+                            string storageRootPath = storageRootDir.FullName;
+                            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
 
-        await logger.ExecuteWithCatchAsync(
-            async ValueTask () =>
-            {
-                if (lastN is 0)
-                {
-                    logger.LogInformation("Deleting all results...");
+                            resultStore = new DiskBasedResultStore(storageRootPath);
 
-                    await resultStore.DeleteResultsAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
-                }
-                else
-                {
-                    logger.LogInformation("Deleting all results except the {lastN} most recent ones...", lastN);
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeDisk;
+                        }
+                        else if (endpointUri is not null)
+                        {
+                            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
 
-                    HashSet<string> toPreserve = [];
+                            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+                            resultStore = new AzureStorageResultStore(fsClient);
 
-                    await foreach (string executionName in
-                        resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
-                    {
-                        _ = toPreserve.Add(executionName);
-                    }
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeAzure;
+                        }
+                        else
+                        {
+                            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+                        }
 
-                    await foreach (string executionName in
-                        resultStore.GetLatestExecutionNamesAsync(
-                            cancellationToken: cancellationToken).ConfigureAwait(false))
-                    {
-                        if (!toPreserve.Contains(executionName))
+                        if (lastN is 0)
                         {
+                            logger.LogInformation("Deleting all results...");
+
                             await resultStore.DeleteResultsAsync(
-                                executionName,
                                 cancellationToken: cancellationToken).ConfigureAwait(false);
                         }
-                    }
-                }
-            }).ConfigureAwait(false);
+                        else
+                        {
+                            logger.LogInformation(
+                                "Deleting all results except the {lastN} most recent ones...",
+                                lastN);
+
+                            HashSet<string> toPreserve = [];
+
+                            await foreach (string executionName in
+                                resultStore.GetLatestExecutionNamesAsync(
+                                    lastN,
+                                    cancellationToken).ConfigureAwait(false))
+                            {
+                                _ = toPreserve.Add(executionName);
+                            }
+
+                            await foreach (string executionName in
+                                resultStore.GetLatestExecutionNamesAsync(
+                                    cancellationToken: cancellationToken).ConfigureAwait(false))
+                            {
+                                if (!toPreserve.Contains(executionName))
+                                {
+                                    await resultStore.DeleteResultsAsync(
+                                        executionName,
+                                        cancellationToken: cancellationToken).ConfigureAwait(false);
+                                }
+                            }
+                        }
+                    },
+                    properties: telemetryProperties,
+                    logger: logger)).ConfigureAwait(false);
 
         return 0;
     }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs
index 2611695e542..cf2242eaac6 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs
@@ -5,19 +5,24 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Identity;
 using Azure.Storage.Files.DataLake;
+using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Html;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Json;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
+using Microsoft.Extensions.AI.Evaluation.Utilities;
 using Microsoft.Extensions.Logging;
+using static Microsoft.Extensions.AI.Evaluation.Console.Telemetry.TelemetryConstants;
 
 namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 
-internal sealed partial class ReportCommand(ILogger logger)
+internal sealed partial class ReportCommand(ILogger logger, TelemetryHelper telemetryHelper)
 {
     internal async Task<int> InvokeAsync(
         DirectoryInfo? storageRootDir,
@@ -28,89 +33,367 @@ internal async Task<int> InvokeAsync(
         Format format,
         CancellationToken cancellationToken = default)
     {
-        IEvaluationResultStore resultStore;
+        var telemetryProperties =
+            new Dictionary<string, string>
+            {
+                [PropertyNames.LastN] = lastN.ToTelemetryPropertyValue(),
+                [PropertyNames.Format] = format.ToString(),
+                [PropertyNames.OpenReport] = openReport.ToTelemetryPropertyValue()
+            };
 
-        if (storageRootDir is not null)
-        {
-            string storageRootPath = storageRootDir.FullName;
-            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+        await logger.ExecuteWithCatchAsync(
+            operation: () =>
+                telemetryHelper.ReportOperationAsync(
+                    operationName: EventNames.ReportCommand,
+                    operation: async ValueTask () =>
+                    {
+                        IEvaluationResultStore resultStore;
 
-            resultStore = new DiskBasedResultStore(storageRootPath);
-        }
-        else if (endpointUri is not null)
-        {
-            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+                        if (storageRootDir is not null)
+                        {
+                            string storageRootPath = storageRootDir.FullName;
+                            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
 
-            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
-            resultStore = new AzureStorageResultStore(fsClient);
-        }
-        else
-        {
-            throw new InvalidOperationException("Either --path or --endpoint must be specified");
-        }
+                            resultStore = new DiskBasedResultStore(storageRootPath);
 
-        List<ScenarioRunResult> results = [];
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeDisk;
+                        }
+                        else if (endpointUri is not null)
+                        {
+                            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
 
-        string? latestExecutionName = null;
+                            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+                            resultStore = new AzureStorageResultStore(fsClient);
 
-        await foreach (string executionName in
-            resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
-        {
-            latestExecutionName ??= executionName;
+                            telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeAzure;
+                        }
+                        else
+                        {
+                            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+                        }
+
+                        List<ScenarioRunResult> results = [];
+                        string? latestExecutionName = null;
+
+                        int resultId = 0;
+                        var usageDetailsByModel =
+                            new Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails>();
+
+                        await foreach (string executionName in
+                            resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
+                        {
+                            latestExecutionName ??= executionName;
+
+                            await foreach (ScenarioRunResult result in
+                                resultStore.ReadResultsAsync(
+                                    executionName,
+                                    cancellationToken: cancellationToken).ConfigureAwait(false))
+                            {
+                                if (result.ExecutionName == latestExecutionName)
+                                {
+                                    ReportScenarioRunResult(
+                                        ++resultId,
+                                        result,
+                                        usageDetailsByModel,
+                                        cancellationToken);
+                                }
+                                else
+                                {
+                                    // Clear the chat data for following executions
+                                    result.Messages = [];
+                                    result.ModelResponse = new ChatResponse();
+                                }
+
+                                results.Add(result);
 
-            await foreach (ScenarioRunResult result in
-                resultStore.ReadResultsAsync(
-                    executionName,
-                    cancellationToken: cancellationToken).ConfigureAwait(false))
+                                logger.LogInformation(
+                                    "Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}",
+                                    result.ExecutionName,
+                                    result.ScenarioName,
+                                    result.IterationName);
+                            }
+                        }
+
+                        ReportUsageDetails(usageDetailsByModel, cancellationToken);
+
+                        string outputFilePath = outputFile.FullName;
+                        string? outputPath = Path.GetDirectoryName(outputFilePath);
+                        if (outputPath is not null && !Directory.Exists(outputPath))
+                        {
+                            _ = Directory.CreateDirectory(outputPath);
+                        }
+
+                        IEvaluationReportWriter reportWriter = format switch
+                        {
+                            Format.html => new HtmlReportWriter(outputFilePath),
+                            Format.json => new JsonReportWriter(outputFilePath),
+                            _ => throw new NotSupportedException(),
+                        };
+
+                        await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false);
+                        logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format);
+
+                        // See the following issues for reasoning behind this check. We want to avoid opening the
+                        // report if this process is running as a service or in a CI pipeline.
+                        // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467
+                        // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289
+                        bool isRedirected =
+                            System.Console.IsInputRedirected &&
+                            System.Console.IsOutputRedirected &&
+                            System.Console.IsErrorRedirected;
+
+                        bool isInteractive =
+                            Environment.UserInteractive && (OperatingSystem.IsWindows() || !isRedirected);
+
+                        if (openReport && isInteractive)
+                        {
+                            // Open the generated report in the default browser.
+                            _ = Process.Start(
+                                new ProcessStartInfo
+                                {
+                                    FileName = outputFilePath,
+                                    UseShellExecute = true
+                                });
+                        }
+                    },
+                    properties: telemetryProperties,
+                    logger: logger)).ConfigureAwait(false);
+
+        return 0;
+    }
+
+    private void ReportScenarioRunResult(
+        int resultId,
+        ScenarioRunResult result,
+        Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails> usageDetailsByModel,
+        CancellationToken cancellationToken)
+    {
+        logger.ExecuteWithCatch(() =>
+        {
+            if (result.ChatDetails?.TurnDetails is IList<ChatTurnDetails> turns)
             {
-                if (result.ExecutionName != latestExecutionName)
+                foreach (ChatTurnDetails turn in turns)
                 {
-                    // Clear the chat data for following executions
-                    result.Messages = [];
-                    result.ModelResponse = new ChatResponse();
+                    cancellationToken.ThrowIfCancellationRequested();
+
+                    (string? model, string? modelProvider) key = (turn.Model, turn.ModelProvider);
+                    if (!usageDetailsByModel.TryGetValue(key, out TurnAndTokenUsageDetails? usageDetails))
+                    {
+                        usageDetails = new TurnAndTokenUsageDetails();
+                        usageDetailsByModel[key] = usageDetails;
+                    }
+
+                    usageDetails.Add(turn);
                 }
+            }
+
+            string resultIdValue = resultId.ToTelemetryPropertyValue();
+            ICollection<EvaluationMetric> metrics = result.EvaluationResult.Metrics.Values;
+
+            var properties =
+                new Dictionary<string, string>
+                {
+                    [PropertyNames.ScenarioRunResultId] = resultIdValue,
+                    [PropertyNames.MetricsCount] = metrics.Count.ToTelemetryPropertyValue()
+                };
 
-                results.Add(result);
+            telemetryHelper.ReportEvent(eventName: EventNames.ScenarioRunResult, properties);
 
-                logger.LogInformation("Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}", result.ExecutionName, result.ScenarioName, result.IterationName);
+            foreach (EvaluationMetric metric in metrics)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                if (metric.IsBuiltIn())
+                {
+                    ReportBuiltInMetric(metric, resultIdValue);
+                }
             }
-        }
+        },
+        swallowUnhandledExceptions: true); // Log and ignore exceptions encountered when trying to report telemetry.
 
-        string outputFilePath = outputFile.FullName;
-        string? outputPath = Path.GetDirectoryName(outputFilePath);
-        if (outputPath is not null && !Directory.Exists(outputPath))
+        void ReportBuiltInMetric(EvaluationMetric metric, string resultIdValue)
         {
-            _ = Directory.CreateDirectory(outputPath);
+            // We always want to report the diagnostics counts - even when metric.Diagnostics is null. This is because
+            // we know that when metric.Diagnostics is null, that means there were no diagnostics (as opposed to
+            // meaning that diagnostic information was somehow missing or unavailable).
+            int errorDiagnosticsCount =
+                metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Error) ?? 0;
+            int warningDiagnosticsCount =
+                metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Warning) ?? 0;
+            int informationalDiagnosticsCount =
+                metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Informational) ?? 0;
+
+            var properties =
+                new Dictionary<string, string>
+                {
+                    [PropertyNames.MetricName] = metric.Name,
+                    [PropertyNames.ScenarioRunResultId] = resultIdValue,
+                    [PropertyNames.ErrorDiagnosticsCount] = errorDiagnosticsCount.ToTelemetryPropertyValue(),
+                    [PropertyNames.WarningDiagnosticsCount] = warningDiagnosticsCount.ToTelemetryPropertyValue(),
+                    [PropertyNames.InformationalDiagnosticsCount] =
+                        informationalDiagnosticsCount.ToTelemetryPropertyValue()
+                };
+
+            // We want to omit reporting the below properties (such as token counts) when the corresponding metadata is
+            // missing. This is because we know that when the metadata is missing, that means the corresponding
+            // information was not available. For example, it would be wrong to report the token counts as 0 when in
+            // reality the token count information is missing because it was not available as part of the ChatResponse
+            // returned from the IChatClient during evaluation.
+            if (TryGetPropertyValueFromMetadata(BuiltInMetricUtilities.EvalModelMetadataName) is string model)
+            {
+                properties[PropertyNames.Model] = model;
+            }
+
+            if (TryGetPropertyValueFromMetadata(BuiltInMetricUtilities.EvalInputTokensMetadataName)
+                    is string inputTokenCount)
+            {
+                properties[PropertyNames.InputTokenCount] = inputTokenCount;
+            }
+
+            if (TryGetPropertyValueFromMetadata(BuiltInMetricUtilities.EvalOutputTokensMetadataName)
+                    is string outputTokenCount)
+            {
+                properties[PropertyNames.OutputTokenCount] = outputTokenCount;
+            }
+
+            if (TryGetPropertyValueFromMetadata(BuiltInMetricUtilities.EvalDurationMillisecondsMetadataName)
+                    is string durationInMilliseconds)
+            {
+                properties[PropertyNames.DurationInMilliseconds] = durationInMilliseconds;
+            }
+
+            if (metric.Interpretation?.Failed is bool failed)
+            {
+                properties[PropertyNames.IsInterpretedAsFailed] = failed.ToTelemetryPropertyValue();
+            }
+
+            telemetryHelper.ReportEvent(eventName: EventNames.BuiltInMetric, properties);
+
+            string? TryGetPropertyValueFromMetadata(string metadataName)
+            {
+                if ((metric.Metadata?.TryGetValue(metadataName, out string? value)) is not true ||
+                    string.IsNullOrWhiteSpace(value))
+                {
+                    return null;
+                }
+
+                return value;
+            }
         }
+    }
 
-        IEvaluationReportWriter reportWriter = format switch
+    private void ReportUsageDetails(
+        Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails> usageDetailsByModel,
+        CancellationToken cancellationToken)
+    {
+        logger.ExecuteWithCatch(() =>
         {
-            Format.html => new HtmlReportWriter(outputFilePath),
-            Format.json => new JsonReportWriter(outputFilePath),
-            _ => throw new NotSupportedException(),
-        };
-
-        await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false);
-        logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format);
-
-        // See the following issues for reasoning behind this check. We want to avoid opening the report
-        // if this process is running as a service or in a CI pipeline.
-        // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467
-        // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289
-        bool isRedirected = System.Console.IsInputRedirected && System.Console.IsOutputRedirected && System.Console.IsErrorRedirected;
-        bool isInteractive = Environment.UserInteractive && (OperatingSystem.IsWindows() || !(isRedirected));
-
-        if (openReport && isInteractive)
+            foreach (((string? model, string? modelProvider), TurnAndTokenUsageDetails usageDetails)
+                in usageDetailsByModel)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                string isModelHostWellKnown = ModelInfo.IsModelHostWellKnown(modelProvider).ToTelemetryPropertyValue();
+                string isModelHostedLocally = ModelInfo.IsModelHostedLocally(modelProvider).ToTelemetryPropertyValue();
+                string cachedTurnCount = usageDetails.CachedTurnCount.ToTelemetryPropertyValue();
+                string nonCachedTurnCount = usageDetails.NonCachedTurnCount.ToTelemetryPropertyValue();
+
+                var properties =
+                    new Dictionary<string, string>
+                    {
+                        [PropertyNames.Model] = model.ToTelemetryPropertyValue(defaultValue: PropertyValues.Unknown),
+                        [PropertyNames.ModelProvider] =
+                            modelProvider.ToTelemetryPropertyValue(defaultValue: PropertyValues.Unknown),
+                        [PropertyNames.IsModelHostWellKnown] = isModelHostWellKnown,
+                        [PropertyNames.IsModelHostedLocally] = isModelHostedLocally,
+                        [PropertyNames.CachedTurnCount] = cachedTurnCount,
+                        [PropertyNames.NonCachedTurnCount] = nonCachedTurnCount
+                    };
+
+                // We want to omit reporting the below token counts when the information is not available. It would be
+                // wrong to report the token counts as 0 when in reality the token count information is missing because
+                // it was not available as part of the ChatResponses returned from the IChatClients used during
+                // evaluation.
+                if (usageDetails.CachedInputTokenCount is long cachedInputTokenCount)
+                {
+                    properties[PropertyNames.CachedInputTokenCount] = cachedInputTokenCount.ToTelemetryPropertyValue();
+                }
+
+                if (usageDetails.CachedOutputTokenCount is long cachedOutputTokenCount)
+                {
+                    properties[PropertyNames.CachedOutputTokenCount] =
+                        cachedOutputTokenCount.ToTelemetryPropertyValue();
+                }
+
+                if (usageDetails.NonCachedInputTokenCount is long nonCachedInputTokenCount)
+                {
+                    properties[PropertyNames.NonCachedInputTokenCount] =
+                        nonCachedInputTokenCount.ToTelemetryPropertyValue();
+                }
+
+                if (usageDetails.NonCachedOutputTokenCount is long nonCachedOutputTokenCount)
+                {
+                    properties[PropertyNames.NonCachedOutputTokenCount] =
+                        nonCachedOutputTokenCount.ToTelemetryPropertyValue();
+                }
+
+                telemetryHelper.ReportEvent(eventName: EventNames.ModelUsageDetails, properties);
+            }
+        },
+        swallowUnhandledExceptions: true); // Log and ignore exceptions encountered when trying to report telemetry.
+    }
+
+    private sealed class TurnAndTokenUsageDetails
+    {
+        internal long CachedTurnCount { get; private set; }
+        internal long NonCachedTurnCount { get; private set; }
+        internal long? CachedInputTokenCount { get; private set; }
+        internal long? NonCachedInputTokenCount { get; private set; }
+        internal long? CachedOutputTokenCount { get; private set; }
+        internal long? NonCachedOutputTokenCount { get; private set; }
+
+        internal void Add(ChatTurnDetails turn)
         {
-            // Open the generated report in the default browser.
-            _ = Process.Start(
-                new ProcessStartInfo
+            EnsureTokenCountsInitialized();
+
+            bool isCached = turn.CacheHit ?? false;
+            if (isCached)
+            {
+                ++CachedTurnCount;
+                CachedInputTokenCount += turn.Usage?.InputTokenCount;
+                CachedOutputTokenCount += turn.Usage?.OutputTokenCount;
+            }
+            else
+            {
+                ++NonCachedTurnCount;
+                NonCachedInputTokenCount += turn.Usage?.InputTokenCount;
+                NonCachedOutputTokenCount += turn.Usage?.OutputTokenCount;
+            }
+
+            void EnsureTokenCountsInitialized()
+            {
+                // If any turn (for a particular model and model provider combination) contains token usage details, we
+                // initialize both the cumulative cached token counts as well as the cumulative non-cached token counts
+                // (for this model and model provider combination) to 0. This is done so that when all token usage (for
+                // a particular model and model provider combination) is non-cached, we can report the cumulative
+                // cached token counts (for this model and model provider combination) as 0 (as opposed to treating the
+                // cached token counts as unknown and omitting them from the reported event), and vice-versa. The
+                // assumption here is that if any turn (for a particular model and model provider combination) contains
+                // token usage details, then all other turns (for the same model and model provider combination) will
+                // also contain this.
+
+                if (turn.Usage?.InputTokenCount is not null)
                 {
-                    FileName = outputFilePath,
-                    UseShellExecute = true
-                });
-        }
+                    CachedInputTokenCount ??= 0;
+                    NonCachedInputTokenCount ??= 0;
+                }
 
-        return 0;
+                if (turn.Usage?.OutputTokenCount is not null)
+                {
+                    CachedOutputTokenCount ??= 0;
+                    NonCachedOutputTokenCount ??= 0;
+                }
+            }
+        }
     }
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
index cd01336f544..dafcc78f988 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj
@@ -28,8 +28,12 @@
     <BuildInParallel>false</BuildInParallel>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <InjectSharedNumericExtensions>true</InjectSharedNumericExtensions>
+  </PropertyGroup>
+
   <ItemGroup>
-    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInEvaluatorUtilities.cs" Link="Utilities\BuiltInEvaluatorUtilities.cs" />
+    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInMetricUtilities.cs" Link="Utilities\BuiltInMetricUtilities.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\ModelInfo.cs" Link="Utilities\ModelInfo.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\TimingHelper.cs" Link="Utilities\TimingHelper.cs" />
   </ItemGroup>
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs
index 8150f3a4d8c..013c3c2de84 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs
@@ -1,32 +1,50 @@
 ﻿// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#if DEBUG
-using System.Diagnostics;
-#endif
 using System;
 using System.CommandLine;
 using System.CommandLine.Parsing;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.Extensions.AI.Evaluation.Console.Commands;
+using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
 using Microsoft.Extensions.Logging;
 
+#if DEBUG
+using System.Diagnostics;
+#endif
+
 namespace Microsoft.Extensions.AI.Evaluation.Console;
 
 internal sealed class Program
 {
     private const string ShortName = "aieval";
     private const string Name = "Microsoft.Extensions.AI.Evaluation.Console";
-    private const string Banner = $"{Name} [{Constants.Version}]";
+    private const string Banner = $"{Name} ({ShortName}) version {Constants.Version}";
 
 #pragma warning disable EA0014 // Async methods should support cancellation.
     private static async Task<int> Main(string[] args)
 #pragma warning restore EA0014
     {
-        using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddConsole());
+#pragma warning disable CA1303 // Do not pass literals as localized parameters.
+        // Use Console.WriteLine directly instead of ILogger to ensure proper formatting.
+        System.Console.WriteLine(Banner);
+        System.Console.WriteLine();
+#pragma warning restore CA1303
+
+        using ILoggerFactory factory =
+            LoggerFactory.Create(builder =>
+                builder.AddSimpleConsole(options =>
+                {
+                    options.SingleLine = true;
+                }));
+
         ILogger logger = factory.CreateLogger(ShortName);
-        logger.LogInformation("{banner}", Banner);
+        await logger.DisplayTelemetryOptOutMessageIfNeededAsync().ConfigureAwait(false);
+
+#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task.
+        await using var telemetryHelper = new TelemetryHelper(logger);
+#pragma warning restore CA2007
 
         var rootCmd = new RootCommand(Banner);
 
@@ -97,7 +115,9 @@ private static async Task<int> Main(string[] args)
         reportCmd.AddOption(formatOpt);
 
         reportCmd.SetHandler(
-            (path, endpoint, output, openReport, lastN, format) => new ReportCommand(logger).InvokeAsync(path, endpoint, output, openReport, lastN, format),
+            (path, endpoint, output, openReport, lastN, format) =>
+                new ReportCommand(logger, telemetryHelper)
+                    .InvokeAsync(path, endpoint, output, openReport, lastN, format),
             pathOpt,
             endpointOpt,
             outputOpt,
@@ -109,7 +129,7 @@ private static async Task<int> Main(string[] args)
 
         // TASK: Support more granular filters such as the specific scenario / iteration / execution whose results must
         // be cleaned up.
-        var cleanResultsCmd = new Command("cleanResults", "Delete results");
+        var cleanResultsCmd = new Command("clean-results", "Delete results");
         cleanResultsCmd.AddOption(pathOpt);
         cleanResultsCmd.AddOption(endpointOpt);
         cleanResultsCmd.AddValidator(requiresPathOrEndpoint);
@@ -118,20 +138,21 @@ private static async Task<int> Main(string[] args)
         cleanResultsCmd.AddOption(lastNOpt2);
 
         cleanResultsCmd.SetHandler(
-            (path, endpoint, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, endpoint, lastN),
+            (path, endpoint, lastN) =>
+                new CleanResultsCommand(logger, telemetryHelper).InvokeAsync(path, endpoint, lastN),
             pathOpt,
             endpointOpt,
             lastNOpt2);
 
         rootCmd.Add(cleanResultsCmd);
 
-        var cleanCacheCmd = new Command("cleanCache", "Delete expired cache entries");
+        var cleanCacheCmd = new Command("clean-cache", "Delete expired cache entries");
         cleanCacheCmd.AddOption(pathOpt);
         cleanCacheCmd.AddOption(endpointOpt);
         cleanCacheCmd.AddValidator(requiresPathOrEndpoint);
 
         cleanCacheCmd.SetHandler(
-            (path, endpoint) => new CleanCacheCommand(logger).InvokeAsync(path, endpoint),
+            (path, endpoint) => new CleanCacheCommand(logger, telemetryHelper).InvokeAsync(path, endpoint),
             pathOpt, endpointOpt);
 
         rootCmd.Add(cleanCacheCmd);
@@ -148,7 +169,7 @@ private static async Task<int> Main(string[] args)
         }
 #endif
 
-        return await rootCmd.InvokeAsync(args).ConfigureAwait(false);
+        int exitCode = await rootCmd.InvokeAsync(args).ConfigureAwait(false);
+        return exitCode;
     }
-
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/TELEMETRY.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/TELEMETRY.md
new file mode 100644
index 00000000000..7db74c24036
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/TELEMETRY.md
@@ -0,0 +1,73 @@
+# Telemetry in the `aieval` .NET tool
+
+## Overview
+
+The `aieval` .NET tool (Microsoft.Extensions.AI.Evaluation.Console) collects anonymous usage data to help us understand how the tool is being used and to improve your experience. This document describes the telemetry data collected, how to opt out, and our commitment to your privacy.
+
+## When is telemetry collected?
+
+Telemetry is collected only when you use the **`aieval` command-line .NET tool** for operations such as:
+- Generating evaluation reports (`dotnet aieval report`)
+- Cleaning cached evaluation results (`dotnet aieval clean-cache`)
+- Cleaning stored evaluation results (`dotnet aieval clean-results`)
+
+Note that using the Microsoft.Extensions.AI.Evaluation libraries in your own code **does not** trigger any telemetry collection. Telemetry is only collected when explicitly using the `aieval` command-line .NET tool.
+
+## What data is collected?
+
+The tool collects anonymous usage information to help us understand how the tool and the evaluation libraries are being used, and to help us identify areas for improvement.
+
+Below are some examples of the kinds of data collected:
+
+### System and environment details
+- Version of the `aieval` .NET tool
+- .NET runtime version
+- Operating system platform and version
+- Whether the tool was running in a CI environment
+
+### Command usage
+- Which tool commands were executed (`report`, `clean-cache`, `clean-results`)
+- Command execution status (success v/s failure)
+- Command execution duration
+- Whether the persisted evaluation data for report generation was stored on local disk v/s azure blob storage
+- Report generation format (e.g., HTML, JSON)
+
+### Evaluation functionality used
+- Number of scenarios evaluated
+- Whether or not the response caching functionality was utilized during evaluation
+- Names of built-in evaluation metrics used (e.g., Coherence, Fluency, Groundedness) and whether any errors were encountered when computing these metrics
+- Model used for evaluation and (input and output) token counts
+- Evaluation duration
+
+## What data is NOT collected?
+
+We are committed to protecting your privacy. All telemetry collected by the `aieval` .NET tool is anonymous and does not contain any personally identifiable information. Additionally, the telemetry collected by the `aieval` .NET tool **does not** include any information that falls into the following categories:
+
+- **Your prompts or conversation content** - The content of your LLM prompts, conversation histories, tool calls, and AI responses that you are evaluating
+- **Contextual data passed to evaluators** - Any additional data you provide as input to the evaluation process
+- **Custom evaluator implementation details** - Code or logic from your custom evaluators (including details pertaining to your own custom evaluation metrics)
+- **File paths or names** - Local file system paths or file names from your projects
+- **Any other personal or sensitive information** - Personal data, credentials, usernames, email addresses, URLs, or any other sensitive information
+
+## How to opt out
+
+You can opt out of telemetry collection by setting the `DOTNET_AIEVAL_TELEMETRY_OPTOUT` environment variable to `1` or `true`.
+
+## First-run experience
+
+The first time you run a particular version of the `aieval` .NET tool, you will see a message informing you about telemetry collection and how to opt out.
+
+```
+---------
+Telemetry
+---------
+The aieval .NET tool collects usage data in order to help us improve your experience. The data is anonymous and doesn't include personal information. You can opt-out of this data collection by setting the DOTNET_AIEVAL_TELEMETRY_OPTOUT environment variable to '1' or 'true' using your favorite shell.
+```
+
+To skip the above first-run telemetry notification (for example, in automated environments), you can set the `DOTNET_AIEVAL_SKIP_FIRST_TIME_EXPERIENCE` environment variable to `1` or `true`. Note that this only suppresses the notification message; telemetry will still be collected unless you also set the `DOTNET_AIEVAL_TELEMETRY_OPTOUT` environment variable.
+
+## Data handling and privacy
+
+- All telemetry data collected is anonymous and does not contain personally identifiable information.
+- The data is used to improve the Microsoft.Extensions.AI.Evaluation libraries and the `aieval` .NET tool.
+- The data is sent securely to Microsoft servers and managed in accordance with Microsoft's privacy policies.
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs
new file mode 100644
index 00000000000..4088d265250
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs
@@ -0,0 +1,165 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using Microsoft.Extensions.Logging;
+using Microsoft.Win32;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+
+// Note: The below code is based on the code in the following file in the dotnet CLI:
+// https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/Telemetry/DevDeviceIDGetter.cs.
+//
+// The logic below should be kept in sync with the code linked above to ensure that the device ID remains consistent
+// across tools.
+
+internal sealed class DeviceIdHelper
+{
+    private const string RegistryKeyPath = @"SOFTWARE\Microsoft\DeveloperTools";
+    private const string RegistryValueName = "deviceid";
+    private const string CacheFileName = "deviceid";
+
+    private static string? _deviceId;
+
+    private readonly ILogger _logger;
+
+    internal DeviceIdHelper(ILogger logger)
+    {
+        _logger = logger;
+    }
+
+    internal string GetDeviceId()
+    {
+        string? deviceId = GetCachedDeviceId();
+
+        if (string.IsNullOrWhiteSpace(deviceId))
+        {
+#pragma warning disable CA1308 // Normalize strings to uppercase.
+            // The DevDeviceId must follow the format specified below.
+            // 1. The value is a randomly generated Guid/ UUID.
+            // 2. The value follows the 8-4-4-4-12 format (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx).
+            // 3. The value shall be all lowercase and only contain hyphens. No braces or brackets.
+            deviceId = Guid.NewGuid().ToString("D").ToLowerInvariant();
+#pragma warning restore CA1308
+
+            try
+            {
+                CacheDeviceId(deviceId);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to cache device ID.");
+
+                // If caching fails, return empty string to avoid reporting a non-cached id.
+                deviceId = string.Empty;
+            }
+        }
+
+        return deviceId;
+    }
+
+    private static string? GetCachedDeviceId()
+    {
+        if (_deviceId is not null)
+        {
+            return _deviceId;
+        }
+
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            using RegistryKey baseKey = RegistryKey.OpenBaseKey(RegistryHive.CurrentUser, RegistryView.Registry64);
+            using RegistryKey? key = baseKey.OpenSubKey(RegistryKeyPath);
+            _deviceId = key?.GetValue(RegistryValueName) as string;
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            string cacheFileDirectoryPath = GetCacheFileDirectoryPathForLinux();
+            ReadCacheFile(cacheFileDirectoryPath);
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            string cacheFileDirectoryPath = GetCacheFileDirectoryPathForMacOS();
+            ReadCacheFile(cacheFileDirectoryPath);
+        }
+
+        return _deviceId;
+
+        static void ReadCacheFile(string cacheFileDirectoryPath)
+        {
+            string cacheFilePath = Path.Combine(cacheFileDirectoryPath, CacheFileName);
+            if (File.Exists(cacheFilePath))
+            {
+                _deviceId = File.ReadAllText(cacheFilePath);
+            }
+        }
+    }
+
+    private static void CacheDeviceId(string deviceId)
+    {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            using RegistryKey baseKey = RegistryKey.OpenBaseKey(RegistryHive.CurrentUser, RegistryView.Registry64);
+            using RegistryKey key = baseKey.CreateSubKey(RegistryKeyPath);
+            key.SetValue(RegistryValueName, deviceId);
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            string cacheFileDirectoryPath = GetCacheFileDirectoryPathForLinux();
+            WriteCacheFile(cacheFileDirectoryPath, deviceId);
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            string cacheFileDirectoryPath = GetCacheFileDirectoryPathForMacOS();
+            WriteCacheFile(cacheFileDirectoryPath, deviceId);
+        }
+
+        _deviceId = deviceId;
+
+        static void WriteCacheFile(string cacheFileDirectoryPath, string deviceId)
+        {
+            _ = Directory.CreateDirectory(cacheFileDirectoryPath);
+            string cacheFilePath = Path.Combine(cacheFileDirectoryPath, CacheFileName);
+            File.WriteAllText(cacheFilePath, deviceId);
+        }
+    }
+
+    private static string GetCacheFileDirectoryPathForLinux()
+    {
+        if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            throw new InvalidOperationException();
+        }
+
+        string cacheFileDirectoryPath;
+        string? xdgCacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME");
+
+        if (string.IsNullOrWhiteSpace(xdgCacheHome))
+        {
+            string userProfilePath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+            cacheFileDirectoryPath = Path.Combine(userProfilePath, ".cache");
+        }
+        else
+        {
+            cacheFileDirectoryPath = Path.Combine(xdgCacheHome, "Microsoft", "DeveloperTools");
+        }
+
+        return cacheFileDirectoryPath;
+    }
+
+    private static string GetCacheFileDirectoryPathForMacOS()
+    {
+        if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            throw new InvalidOperationException();
+        }
+
+        string userProfilePath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+
+        string cacheFileDirectoryPath =
+            Path.Combine(userProfilePath, "Library", "Application Support", "Microsoft", "DeveloperTools");
+
+        return cacheFileDirectoryPath;
+    }
+}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs
new file mode 100644
index 00000000000..28942cb4eb1
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs
@@ -0,0 +1,65 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Linq;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+
+// Note: The below code is based on the code in the following file in the dotnet CLI:
+// https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/Telemetry/CIEnvironmentDetectorForTelemetry.cs.
+//
+// The logic below should be kept in sync with the code linked above.
+
+internal static class EnvironmentHelper
+{
+    internal static bool GetEnvironmentVariableAsBoolean(string name) =>
+        Environment.GetEnvironmentVariable(name)?.ToUpperInvariant() switch
+        {
+            "TRUE" or "1" or "YES" => true,
+            _ => false
+        };
+
+    // CI systems that can be detected via an environment variable with boolean value true.
+    private static readonly string[] _mustBeTrueCIVariables =
+        [
+            "CI",             // A general-use flag supported by many of the major CI systems including: Azure DevOps, GitHub, GitLab, AppVeyor, Travis CI, CircleCI.
+            "TF_BUILD",       // https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables-devops-services
+            "GITHUB_ACTIONS", // https://docs.github.com/en/actions/reference/workflows-and-actions/variables
+            "APPVEYOR",       // https://www.appveyor.com/docs/environment-variables/
+            "TRAVIS",         // https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
+            "CIRCLECI",       // https://circleci.com/docs/reference/variables/#built-in-environment-variables
+        ];
+
+    // CI systems that that can be detected via a set of environment variables where every variable must be present and
+    // must have a non-null value.
+    private static readonly string[][] _mustNotBeNullCIVariables =
+        [
+            ["CODEBUILD_BUILD_ID", "AWS_REGION"], // https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
+            ["BUILD_ID", "BUILD_URL"],            // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/resources/jenkins/model/CoreEnvironmentContributor/buildEnv.groovy
+            ["BUILD_ID", "PROJECT_ID"],           // https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values#using_default_substitutions
+            ["TEAMCITY_VERSION"],                 // https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters
+            ["JB_SPACE_API_URL"]                  // https://www.jetbrains.com/help/space/automation-parameters.html#provided-parameters
+        ];
+
+    public static bool IsCIEnvironment()
+    {
+        foreach (string variable in _mustBeTrueCIVariables)
+        {
+            if (bool.TryParse(Environment.GetEnvironmentVariable(variable), out bool value) && value)
+            {
+                return true;
+            }
+        }
+
+        foreach (string[] variables in _mustNotBeNullCIVariables)
+        {
+            if (variables.All(variable => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(variable))))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs
new file mode 100644
index 00000000000..6a6d5549e07
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs
@@ -0,0 +1,174 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+
+internal static class TelemetryConstants
+{
+    internal const string ConnectionString = "InstrumentationKey=469489a6-628b-4bb9-80db-ec670f70d874";
+
+    internal const string EventNamespace = "dotnet/aieval";
+
+    internal static class EventNames
+    {
+        internal const string CleanCacheCommand = "CleanCacheCommand";
+        internal const string CleanResultsCommand = "CleanResultsCommand";
+        internal const string ReportCommand = "ReportCommand";
+
+        internal const string ScenarioRunResult = "ScenarioRunResult";
+        internal const string BuiltInMetric = "BuiltInMetric";
+        internal const string ModelUsageDetails = "ModelUsageDetails";
+    }
+
+    internal static class PropertyNames
+    {
+        // Properties common to all events.
+        internal const string DevDeviceId = "DevDeviceId";
+        internal const string OSVersion = "OSVersion";
+        internal const string OSPlatform = "OSPlatform";
+        internal const string KernelVersion = "KernelVersion";
+        internal const string RuntimeId = "RuntimeId";
+        internal const string ProductVersion = "ProductVersion";
+        internal const string IsCIEnvironment = "IsCIEnvironment";
+
+        // Properties common to all *Command events.
+        internal const string Success = "Success";
+        internal const string DurationInMilliseconds = "DurationInMilliseconds";
+
+        // Properties for parameters included in corresponding *Command events.
+        internal const string StorageType = "StorageType";
+        internal const string LastN = "LastN";
+        internal const string Format = "Format";
+        internal const string OpenReport = "OpenReport";
+
+        // Properties included in the ScenarioRunResult event.
+        internal const string ScenarioRunResultId = "ScenarioRunResultId";
+        internal const string MetricsCount = "MetricsCount";
+
+        // Properties included in the BuiltInMetric event.
+        internal const string MetricName = "MetricName";
+        internal const string InputTokenCount = "InputTokenCount";
+        internal const string OutputTokenCount = "OutputTokenCount";
+        internal const string IsInterpretedAsFailed = "IsInterpretedAsFailed";
+        internal const string ErrorDiagnosticsCount = "ErrorDiagnosticsCount";
+        internal const string WarningDiagnosticsCount = "WarningDiagnosticsCount";
+        internal const string InformationalDiagnosticsCount = "InformationalDiagnosticsCount";
+
+        // Properties included in the ModelUsageDetails event.
+        internal const string Model = "Model";
+        internal const string ModelProvider = "ModelProvider";
+        internal const string IsModelHostWellKnown = "IsModelHostWellKnown";
+        internal const string IsModelHostedLocally = "IsModelHostedLocally";
+        internal const string CachedTurnCount = "CachedTurnCount";
+        internal const string NonCachedTurnCount = "NonCachedTurnCount";
+        internal const string CachedInputTokenCount = "CachedInputTokenCount";
+        internal const string NonCachedInputTokenCount = "NonCachedInputTokenCount";
+        internal const string CachedOutputTokenCount = "CachedOutputTokenCount";
+        internal const string NonCachedOutputTokenCount = "NonCachedOutputTokenCount";
+    }
+
+    internal static class PropertyValues
+    {
+        internal const string Unknown = "unknown";
+
+        internal const string StorageTypeDisk = "disk";
+        internal const string StorageTypeAzure = "azure";
+
+        // Kusto recongnizes "true" and "false" as boolean literals.
+        internal const string True = "true";
+        internal const string False = "false";
+    }
+
+    private const string TelemetryOptOutEnvironmentVariableName = "DOTNET_AIEVAL_TELEMETRY_OPTOUT";
+    private const string SkipFirstTimeExperienceEnvironmentVariableName = "DOTNET_AIEVAL_SKIP_FIRST_TIME_EXPERIENCE";
+    private const string TelemetryOptOutMessage =
+        $"""
+        ---------
+        Telemetry
+        ---------
+        The aieval .NET tool collects usage data in order to help us improve your experience. The data is anonymous and doesn't include personal information. You can opt-out of this data collection by setting the {TelemetryOptOutEnvironmentVariableName} environment variable to '1' or 'true' using your favorite shell.
+        """;
+
+    private static readonly string? _firstUseSentinelFilePath;
+    private static readonly bool _firstUseSentinelFileExists;
+    private static readonly bool _shouldDisplayTelemetryOptOutMessage;
+
+    internal static bool IsTelemetryEnabled { get; }
+
+#pragma warning disable CA1810
+    // CA1810: Initialize all static fields when declared.
+    // We disable this warning because the static fields above must be initialized in a specific order which is not
+    // guaranteed when initializing via field initializers.
+    static TelemetryConstants()
+#pragma warning restore CA1810
+    {
+        _firstUseSentinelFilePath = GetFirstUseSentinelFilePath();
+        _firstUseSentinelFileExists = File.Exists(_firstUseSentinelFilePath);
+
+        _shouldDisplayTelemetryOptOutMessage =
+            !EnvironmentHelper.GetEnvironmentVariableAsBoolean(SkipFirstTimeExperienceEnvironmentVariableName) &&
+            !_firstUseSentinelFileExists;
+
+        IsTelemetryEnabled =
+            !EnvironmentHelper.GetEnvironmentVariableAsBoolean(TelemetryOptOutEnvironmentVariableName) &&
+            !_shouldDisplayTelemetryOptOutMessage;
+
+        static string? GetFirstUseSentinelFilePath()
+        {
+            string? homeDirectoryPath = Environment.GetEnvironmentVariable("DOTNET_CLI_HOME");
+            if (string.IsNullOrWhiteSpace(homeDirectoryPath))
+            {
+                homeDirectoryPath =
+                    Environment.GetEnvironmentVariable(
+                        RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "USERPROFILE" : "HOME");
+            }
+
+            string? sentinelFilePath =
+                string.IsNullOrWhiteSpace(homeDirectoryPath)
+                    ? null
+                    : Path.Combine(homeDirectoryPath, ".dotnet", $"{Constants.Version}.aieval.dotnetFirstUseSentinel");
+
+            return sentinelFilePath;
+        }
+    }
+
+#pragma warning disable EA0014 // The async method doesn't support cancellation.
+    internal static async Task DisplayTelemetryOptOutMessageIfNeededAsync(this ILogger logger)
+#pragma warning restore EA0014
+    {
+        if (_shouldDisplayTelemetryOptOutMessage)
+        {
+#pragma warning disable CA1303 // Do not pass literals as localized parameters.
+            // Use Console.WriteLine directly instead of ILogger to ensure proper formatting.
+            System.Console.WriteLine(TelemetryOptOutMessage);
+            System.Console.WriteLine();
+#pragma warning restore CA1303
+        }
+
+        if (_firstUseSentinelFilePath is null)
+        {
+            logger.LogWarning("Could not determine sentinel file path.");
+            return;
+        }
+
+        if (_firstUseSentinelFileExists)
+        {
+            return;
+        }
+
+        try
+        {
+            await File.WriteAllBytesAsync(_firstUseSentinelFilePath, []).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            logger.LogWarning(ex, "Failed to create sentinel file.");
+        }
+    }
+}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs
new file mode 100644
index 00000000000..685e7153053
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs
@@ -0,0 +1,228 @@
+﻿// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
+using Microsoft.Extensions.AI.Evaluation.Utilities;
+using Microsoft.Extensions.Logging;
+using Microsoft.Shared.Text;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+
+internal static class TelemetryExtensions
+{
+    internal static string ToTelemetryPropertyValue(this bool value) =>
+        value ? TelemetryConstants.PropertyValues.True : TelemetryConstants.PropertyValues.False;
+
+    internal static string ToTelemetryPropertyValue(this int value) =>
+        value.ToInvariantString();
+
+    internal static string ToTelemetryPropertyValue(this long value) =>
+        value.ToInvariantString();
+
+    internal static string ToTelemetryPropertyValue(this string? value, string defaultValue) =>
+        string.IsNullOrWhiteSpace(value) ? defaultValue : value;
+
+    internal static void ReportOperation(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Action operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            TimeSpan duration = TimingHelper.ExecuteWithTiming(operation);
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+    internal static TResult ReportOperation<TResult>(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Func<TResult> operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            (TResult result, TimeSpan duration) = TimingHelper.ExecuteWithTiming(operation);
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+            return result;
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+#pragma warning disable EA0014 // The async method doesn't support cancellation.
+    internal static async ValueTask ReportOperationAsync(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Func<Task> operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            TimeSpan duration = await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false);
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+    internal static async ValueTask ReportOperationAsync(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Func<ValueTask> operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            TimeSpan duration = await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false);
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+    internal static async ValueTask<TResult> ReportOperationAsync<TResult>(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Func<Task<TResult>> operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            (TResult result, TimeSpan duration) =
+                await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false);
+
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+            return result;
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+    internal static async ValueTask<TResult> ReportOperationAsync<TResult>(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Func<ValueTask<TResult>> operation,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        try
+        {
+            (TResult result, TimeSpan duration) =
+                await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false);
+
+            telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics, logger);
+            return result;
+        }
+        catch (Exception ex)
+        {
+            telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics, logger);
+            throw;
+        }
+    }
+
+    private static void ReportOperationSuccess(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        TimeSpan duration,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        void Report()
+        {
+            string durationInMilliseconds = duration.ToMillisecondsText();
+
+            properties ??= new Dictionary<string, string>();
+            properties.Add(TelemetryConstants.PropertyNames.Success, TelemetryConstants.PropertyValues.True);
+            properties.Add(TelemetryConstants.PropertyNames.DurationInMilliseconds, durationInMilliseconds);
+
+            telemetryHelper.ReportEvent(eventName: operationName, properties, metrics);
+        }
+
+        if (logger is null)
+        {
+            try
+            {
+                Report();
+            }
+            catch
+            {
+                // Ignore exceptions encountered when trying to report telemetry.
+            }
+        }
+        else
+        {
+            // Log and ignore exceptions encountered when trying to report telemetry.
+            logger.ExecuteWithCatch(Report, swallowUnhandledExceptions: true);
+        }
+    }
+
+    private static void ReportOperationFailure(
+        this TelemetryHelper telemetryHelper,
+        string operationName,
+        Exception exception,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null,
+        ILogger? logger = null)
+    {
+        void Report()
+        {
+            properties ??= new Dictionary<string, string>();
+            properties.Add(TelemetryConstants.PropertyNames.Success, TelemetryConstants.PropertyValues.False);
+
+            telemetryHelper.ReportEvent(eventName: operationName, properties, metrics);
+            telemetryHelper.ReportException(exception, properties, metrics);
+        }
+
+        if (logger is null)
+        {
+            try
+            {
+                Report();
+            }
+            catch
+            {
+                // Ignore exceptions encountered when trying to report telemetry.
+            }
+        }
+        else
+        {
+            // Log and ignore exceptions encountered when trying to report telemetry.
+            logger.ExecuteWithCatch(Report, swallowUnhandledExceptions: true);
+        }
+    }
+#pragma warning restore EA0014
+}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs
new file mode 100644
index 00000000000..f72822e150b
--- /dev/null
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs
@@ -0,0 +1,184 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ApplicationInsights;
+using Microsoft.ApplicationInsights.Extensibility;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
+
+internal sealed class TelemetryHelper : IAsyncDisposable
+{
+    private readonly ILogger _logger;
+    private readonly TelemetryConfiguration? _telemetryConfiguration;
+    private readonly TelemetryClient? _telemetryClient;
+    private readonly Dictionary<string, string>? _commonProperties;
+    private bool _disposed;
+
+    [MemberNotNullWhen(false, nameof(_telemetryConfiguration), nameof(_telemetryClient), nameof(_commonProperties))]
+    private bool Disabled { get; }
+
+    internal TelemetryHelper(ILogger logger)
+    {
+        _logger = logger;
+
+        if (!TelemetryConstants.IsTelemetryEnabled)
+        {
+            Disabled = true;
+            return;
+        }
+
+        try
+        {
+            _telemetryConfiguration = TelemetryConfiguration.CreateDefault();
+            _telemetryConfiguration.ConnectionString = TelemetryConstants.ConnectionString;
+            _telemetryClient = new TelemetryClient(_telemetryConfiguration);
+
+            var deviceIdHelper = new DeviceIdHelper(logger);
+            string deviceId = deviceIdHelper.GetDeviceId();
+            string isCIEnvironment = EnvironmentHelper.IsCIEnvironment().ToTelemetryPropertyValue();
+
+            _commonProperties =
+                new Dictionary<string, string>
+                {
+                    [TelemetryConstants.PropertyNames.DevDeviceId] = deviceId,
+                    [TelemetryConstants.PropertyNames.OSVersion] = Environment.OSVersion.VersionString,
+                    [TelemetryConstants.PropertyNames.OSPlatform] = Environment.OSVersion.Platform.ToString(),
+                    [TelemetryConstants.PropertyNames.KernelVersion] = RuntimeInformation.OSDescription,
+                    [TelemetryConstants.PropertyNames.RuntimeId] = RuntimeInformation.RuntimeIdentifier,
+                    [TelemetryConstants.PropertyNames.ProductVersion] = Constants.Version,
+                    [TelemetryConstants.PropertyNames.IsCIEnvironment] = isCIEnvironment
+                };
+
+            _telemetryClient.Context.Session.Id = Guid.NewGuid().ToString();
+            _telemetryClient.Context.Device.OperatingSystem = RuntimeInformation.OSDescription;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, $"Failed to initialize {nameof(TelemetryHelper)}.");
+
+            _telemetryConfiguration?.Dispose();
+            _telemetryConfiguration = null;
+            _telemetryClient = null;
+            _commonProperties = null;
+
+            Disabled = true;
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        if (Disabled || _disposed)
+        {
+            return;
+        }
+
+        try
+        {
+            _ = await FlushAsync().ConfigureAwait(false);
+            _telemetryConfiguration.Dispose();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, $"Failed to dispose {nameof(TelemetryHelper)}.");
+        }
+
+        _disposed = true;
+    }
+
+    internal void ReportEvent(
+        string eventName,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null)
+    {
+        if (Disabled || _disposed)
+        {
+            return;
+        }
+
+        try
+        {
+            IDictionary<string, string>? combinedProperties = GetCombinedProperties(properties);
+
+            _telemetryClient.TrackEvent(
+                $"{TelemetryConstants.EventNamespace}/{eventName}",
+                combinedProperties,
+                metrics);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, $"Failed to report event '{eventName}' in telemetry.");
+        }
+    }
+
+    internal void ReportException(
+        Exception exception,
+        IDictionary<string, string>? properties = null,
+        IDictionary<string, double>? metrics = null)
+    {
+        if (Disabled || _disposed)
+        {
+            return;
+        }
+
+        try
+        {
+            IDictionary<string, string>? combinedProperties = GetCombinedProperties(properties);
+
+            _telemetryClient.TrackException(exception, combinedProperties, metrics);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, $"Failed to report exception in telemetry.");
+        }
+    }
+
+    internal async Task<bool> FlushAsync(CancellationToken cancellationToken = default)
+    {
+        if (Disabled || _disposed)
+        {
+            return false;
+        }
+
+        try
+        {
+            return await _telemetryClient.FlushAsync(cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to flush telemetry.");
+
+            return false;
+        }
+    }
+
+    private Dictionary<string, string>? GetCombinedProperties(IDictionary<string, string>? properties)
+    {
+        if (Disabled || _disposed)
+        {
+            return null;
+        }
+
+        Dictionary<string, string> combinedProperties;
+        if (properties is null)
+        {
+            combinedProperties = _commonProperties;
+        }
+        else
+        {
+            combinedProperties = new Dictionary<string, string>(_commonProperties);
+            foreach (var kvp in properties)
+            {
+                combinedProperties.Add(kvp.Key, kvp.Value);
+            }
+        }
+
+        return combinedProperties;
+    }
+}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Utilities/LoggerExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Utilities/LoggerExtensions.cs
index 4ff77f242d1..f18ef6c1f4a 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Utilities/LoggerExtensions.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Utilities/LoggerExtensions.cs
@@ -24,10 +24,6 @@ internal static void ExecuteWithCatch(
         {
             operation();
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
@@ -44,10 +40,6 @@ internal static void ExecuteWithCatch(
         {
             return operation();
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
@@ -56,7 +48,7 @@ internal static void ExecuteWithCatch(
         return defaultValue;
     }
 
-#pragma warning disable EA0014 // The async method doesn't support cancellation
+#pragma warning disable EA0014 // The async method doesn't support cancellation.
     internal static async ValueTask ExecuteWithCatchAsync(
         this ILogger logger,
         Func<Task> operation,
@@ -66,10 +58,6 @@ internal static async ValueTask ExecuteWithCatchAsync(
         {
             await operation().ConfigureAwait(false);
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
@@ -85,10 +73,6 @@ internal static async ValueTask ExecuteWithCatchAsync(
         {
             await operation().ConfigureAwait(false);
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
@@ -105,10 +89,6 @@ internal static async ValueTask ExecuteWithCatchAsync(
         {
             return await operation().ConfigureAwait(false);
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
@@ -127,10 +107,6 @@ internal static async ValueTask ExecuteWithCatchAsync(
         {
             return await operation().ConfigureAwait(false);
         }
-        catch (Exception ex) when (swallowUnhandledExceptions && ex.IsCancellation())
-        {
-            // Do nothing.
-        }
         catch (Exception ex) when (!ex.IsCancellation() && logger.LogException(ex) && swallowUnhandledExceptions)
         {
             // Do nothing. The exception is logged in the when clause above.
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs
index 5d9df9c801f..1eecbc20510 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -79,16 +78,27 @@ public ValueTask<EvaluationResult> EvaluateAsync(
             return new ValueTask<EvaluationResult>(result);
         }
 
-        (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() =>
-        {
-            string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray();
-            string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
-            return BLEUAlgorithm.SentenceBLEU(references, hypothesis, BLEUAlgorithm.DefaultBLEUWeights, SmoothingFunction.Method4);
-        });
+        (double score, TimeSpan duration) =
+            TimingHelper.ExecuteWithTiming(() =>
+            {
+                string[][] references =
+                    context.References.Select(
+                        reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray();
+
+                string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
+
+                double score =
+                    BLEUAlgorithm.SentenceBLEU(
+                        references,
+                        hypothesis,
+                        BLEUAlgorithm.DefaultBLEUWeights,
+                        SmoothingFunction.Method4);
+
+                return score;
+            });
 
         metric.Value = score;
-        string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s";
-        metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText);
+        metric.AddOrUpdateDurationMetadata(duration);
         metric.AddOrUpdateContext(context);
         metric.Interpretation = metric.Interpret();
 
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs
index 6924524ffb8..52519e0350f 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -70,16 +69,17 @@ public ValueTask<EvaluationResult> EvaluateAsync(
             return new ValueTask<EvaluationResult>(result);
         }
 
-        (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() =>
-        {
-            string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray();
-            string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
-            return F1Algorithm.CalculateF1Score(reference, hypothesis);
-        });
+        (double score, TimeSpan duration) =
+            TimingHelper.ExecuteWithTiming(() =>
+            {
+                string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray();
+                string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
+                double score = F1Algorithm.CalculateF1Score(reference, hypothesis);
+                return score;
+            });
 
         metric.Value = score;
-        string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s";
-        metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText);
+        metric.AddOrUpdateDurationMetadata(duration);
         metric.AddOrUpdateContext(context);
         metric.Interpretation = metric.Interpret();
 
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs
index fb32b6c81bd..04282a6164d 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -79,16 +78,20 @@ public ValueTask<EvaluationResult> EvaluateAsync(
             return new ValueTask<EvaluationResult>(result);
         }
 
-        (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() =>
-        {
-            string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray();
-            string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
-            return GLEUAlgorithm.SentenceGLEU(references, hypothesis);
-        });
+        (double score, TimeSpan duration) =
+            TimingHelper.ExecuteWithTiming(() =>
+            {
+                string[][] references = context.References.Select(
+                    reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray();
+
+                string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
+
+                double score = GLEUAlgorithm.SentenceGLEU(references, hypothesis);
+                return score;
+            });
 
         metric.Value = score;
-        string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s";
-        metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText);
+        metric.AddOrUpdateDurationMetadata(duration);
         metric.AddOrUpdateContext(context);
         metric.Interpretation = metric.Interpret();
 
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj
index c6d52244f87..7f77f5ce397 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj
@@ -21,7 +21,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInEvaluatorUtilities.cs" Link="Utilities\BuiltInEvaluatorUtilities.cs" />
+    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInMetricUtilities.cs" Link="Utilities\BuiltInMetricUtilities.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\TimingHelper.cs" Link="Utilities\TimingHelper.cs" />
   </ItemGroup>
 
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj
index 50fa41b7549..a80aa4951bc 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj
@@ -20,7 +20,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInEvaluatorUtilities.cs" Link="Utilities\BuiltInEvaluatorUtilities.cs" />
+    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInMetricUtilities.cs" Link="Utilities\BuiltInMetricUtilities.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\TimingHelper.cs" Link="Utilities\TimingHelper.cs" />
   </ItemGroup>
 
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs
index e4788bcdc81..0935e5a1f58 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs
@@ -30,7 +30,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety;
 /// <see cref="ViolenceEvaluator.ViolenceMetricName"/>, <see cref="SelfHarmEvaluator.SelfHarmMetricName"/> and
 /// <see cref="SexualEvaluator.SexualMetricName"/>.
 /// </param>
-#pragma warning disable SA1118 // Parameter should not span multiple lines
+#pragma warning disable SA1118 // Parameter should not span multiple lines.
 public class ContentHarmEvaluator(IDictionary<string, string>? metricNames = null)
     : ContentSafetyEvaluator(
         contentSafetyServiceAnnotationTask: "content harm",
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs
index d5414f47741..5f173bfb0b3 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs
@@ -173,7 +173,7 @@ EvaluationResult UpdateMetrics()
                     metric.AddDiagnostics(diagnostics);
                 }
 
-#pragma warning disable S125 // Sections of code should not be commented out
+#pragma warning disable S125 // Sections of code should not be commented out.
                 // The following commented code can be useful for debugging purposes.
                 // metric.LogJsonData(payload);
                 // metric.LogJsonData(annotationResult);
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs
index 26dec553915..3f81cdbc1e6 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs
@@ -34,7 +34,7 @@ internal sealed partial class ContentSafetyService(ContentSafetyServiceConfigura
 
     internal static EvaluationResult ParseAnnotationResult(string annotationResponse)
     {
-#pragma warning disable S125 // Sections of code should not be commented out
+#pragma warning disable S125 // Sections of code should not be commented out.
         // Example annotation response:
         // [
         //   {
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj
index b1ac49dce28..8cfc9e1b538 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj
@@ -17,7 +17,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInEvaluatorUtilities.cs" Link="Utilities\BuiltInEvaluatorUtilities.cs" />
+    <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\BuiltInMetricUtilities.cs" Link="Utilities\BuiltInMetricUtilities.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\ModelInfo.cs" Link="Utilities\ModelInfo.cs" />
     <Compile Include="..\Microsoft.Extensions.AI.Evaluation\Utilities\TimingHelper.cs" Link="Utilities\TimingHelper.cs" />
   </ItemGroup>
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs
index 534f5e300f7..1940ee562d5 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs
@@ -3,9 +3,10 @@
 
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.Linq;
+using Microsoft.Extensions.AI.Evaluation.Utilities;
 using Microsoft.Shared.Diagnostics;
+using Microsoft.Shared.Text;
 
 namespace Microsoft.Extensions.AI.Evaluation;
 
@@ -85,7 +86,7 @@ public static void AddDiagnostics(this EvaluationMetric metric, IEnumerable<Eval
 
         if (diagnostics.Any())
         {
-            metric.Diagnostics ??= new List<EvaluationDiagnostic>();
+            metric.Diagnostics ??= [];
 
             foreach (EvaluationDiagnostic diagnostic in diagnostics)
             {
@@ -143,7 +144,8 @@ public static void AddOrUpdateMetadata(this EvaluationMetric metric, IDictionary
     /// <param name="response">The <see cref="ChatResponse"/> that contains metadata to be added or updated.</param>
     /// <param name="duration">
     /// An optional duration that represents the amount of time that it took for the AI model to produce the supplied
-    /// <paramref name="response"/>. If supplied, the duration will also be included as part of the added metadata.
+    /// <paramref name="response"/>. If supplied, the duration (in milliseconds) will also be included as part of the
+    /// added metadata.
     /// </param>
     public static void AddOrUpdateChatMetadata(
         this EvaluationMetric metric,
@@ -154,31 +156,52 @@ public static void AddOrUpdateChatMetadata(
 
         if (!string.IsNullOrWhiteSpace(response.ModelId))
         {
-            metric.AddOrUpdateMetadata(name: "evaluation-model-used", value: response.ModelId!);
+            metric.AddOrUpdateMetadata(name: BuiltInMetricUtilities.EvalModelMetadataName, value: response.ModelId!);
         }
 
         if (response.Usage is UsageDetails usage)
         {
             if (usage.InputTokenCount is not null)
             {
-                metric.AddOrUpdateMetadata(name: "evaluation-input-tokens-used", value: $"{usage.InputTokenCount}");
+                metric.AddOrUpdateMetadata(
+                    name: BuiltInMetricUtilities.EvalInputTokensMetadataName,
+                    value: usage.InputTokenCount.Value.ToInvariantString());
             }
 
             if (usage.OutputTokenCount is not null)
             {
-                metric.AddOrUpdateMetadata(name: "evaluation-output-tokens-used", value: $"{usage.OutputTokenCount}");
+                metric.AddOrUpdateMetadata(
+                    name: BuiltInMetricUtilities.EvalOutputTokensMetadataName,
+                    value: usage.OutputTokenCount.Value.ToInvariantString());
             }
 
             if (usage.TotalTokenCount is not null)
             {
-                metric.AddOrUpdateMetadata(name: "evaluation-total-tokens-used", value: $"{usage.TotalTokenCount}");
+                metric.AddOrUpdateMetadata(
+                    name: BuiltInMetricUtilities.EvalTotalTokensMetadataName,
+                    value: usage.TotalTokenCount.Value.ToInvariantString());
             }
         }
 
         if (duration is not null)
         {
-            string durationText = $"{duration.Value.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s";
-            metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText);
+            metric.AddOrUpdateDurationMetadata(duration.Value);
         }
     }
+
+    /// <summary>
+    /// Adds or updates metadata identifying the amount of time (in milliseconds) that it took to perform the
+    /// evaluation in the supplied <paramref name="metric"/>'s <see cref="EvaluationMetric.Metadata"/> dictionary.
+    /// </summary>
+    /// <param name="metric">The <see cref="EvaluationMetric"/>.</param>
+    /// <param name="duration">
+    /// The amount of time that it took to perform the evaluation that produced the supplied <paramref name="metric"/>.
+    /// </param>
+    public static void AddOrUpdateDurationMetadata(this EvaluationMetric metric, TimeSpan duration)
+    {
+        string durationInMilliseconds = duration.ToMillisecondsText();
+        metric.AddOrUpdateMetadata(
+            name: BuiltInMetricUtilities.EvalDurationMillisecondsMetadataName,
+            value: durationInMilliseconds);
+    }
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs
index 7ea21dec91f..ba199e26de1 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs
@@ -173,7 +173,8 @@ public static void AddOrUpdateMetadataInAllMetrics(
     /// <param name="response">The <see cref="ChatResponse"/> that contains metadata to be added or updated.</param>
     /// <param name="duration">
     /// An optional duration that represents the amount of time that it took for the AI model to produce the supplied
-    /// <paramref name="response"/>. If supplied, the duration will also be included as part of the added metadata.
+    /// <paramref name="response"/>. If supplied, the duration (in milliseconds) will also be included as part of the
+    /// added metadata.
     /// </param>
     public static void AddOrUpdateChatMetadataInAllMetrics(
         this EvaluationResult result,
@@ -187,4 +188,24 @@ public static void AddOrUpdateChatMetadataInAllMetrics(
             metric.AddOrUpdateChatMetadata(response, duration);
         }
     }
+
+    /// <summary>
+    /// Adds or updates metadata identifying the amount of time (in milliseconds) that it took to perform the
+    /// evaluation in all <see cref="EvaluationMetric"/>s contained in the supplied <paramref name="result"/>.
+    /// </summary>
+    /// <param name="result">
+    /// The <see cref="EvaluationResult"/> containing the <see cref="EvaluationMetric"/>s that are to be altered.
+    /// </param>
+    /// <param name="duration">
+    /// The amount of time that it took to perform the evaluation that produced the supplied <paramref name="result"/>.
+    /// </param>
+    public static void AddOrUpdateDurationMetadataInAllMetrics(this EvaluationResult result, TimeSpan duration)
+    {
+        _ = Throw.IfNull(result);
+
+        foreach (EvaluationMetric metric in result.Metrics.Values)
+        {
+            metric.AddOrUpdateDurationMetadata(duration);
+        }
+    }
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj
index 9e9c1eeec3d..129ae2aab89 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj
@@ -14,6 +14,10 @@
     <MinMutationScore>n/a</MinMutationScore>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <InjectSharedNumericExtensions>true</InjectSharedNumericExtensions>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.Extensions.AI.Abstractions\Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json
index 59fc0cad90a..d9b464ef6b4 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json
@@ -259,6 +259,10 @@
           "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateContext(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, params Microsoft.Extensions.AI.Evaluation.EvaluationContext[] context);",
           "Stage": "Stable"
         },
+        {
+          "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateDurationMetadata(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, System.TimeSpan duration);",
+          "Stage": "Stable"
+        },
         {
           "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateMetadata(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, string name, string value);",
           "Stage": "Stable"
@@ -403,6 +407,10 @@
           "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateContextInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, params Microsoft.Extensions.AI.Evaluation.EvaluationContext[] context);",
           "Stage": "Stable"
         },
+        {
+          "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateDurationMetadataInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, System.TimeSpan duration);",
+          "Stage": "Stable"
+        },
         {
           "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateMetadataInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, string name, string value);",
           "Stage": "Stable"
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs
similarity index 52%
rename from src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs
rename to src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs
index a86da7aaf6d..5718cf95a6e 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs
@@ -3,12 +3,17 @@
 
 namespace Microsoft.Extensions.AI.Evaluation.Utilities;
 
-internal static class BuiltInEvaluatorUtilities
+internal static class BuiltInMetricUtilities
 {
+    internal const string EvalModelMetadataName = "eval-model";
+    internal const string EvalInputTokensMetadataName = "eval-input-tokens";
+    internal const string EvalOutputTokensMetadataName = "eval-output-tokens";
+    internal const string EvalTotalTokensMetadataName = "eval-total-tokens";
+    internal const string EvalDurationMillisecondsMetadataName = "eval-duration-ms";
     internal const string BuiltInEvalMetadataName = "built-in-eval";
 
     internal static void MarkAsBuiltIn(this EvaluationMetric metric) =>
-        metric.AddOrUpdateMetadata(BuiltInEvalMetadataName, "True");
+        metric.AddOrUpdateMetadata(name: BuiltInEvalMetadataName, value: bool.TrueString);
 
     internal static bool IsBuiltIn(this EvaluationMetric metric) =>
         metric.Metadata?.TryGetValue(BuiltInEvalMetadataName, out string? stringValue) is true &&
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/ModelInfo.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/ModelInfo.cs
index b5c74212401..483e1fbe77f 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/ModelInfo.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/ModelInfo.cs
@@ -76,7 +76,7 @@ internal static class KnownModelHostMonikers
     /// </param>
     internal static string? GetModelProvider(string? model, ChatClientMetadata? metadata)
     {
-#pragma warning disable S2219 // Runtime type checking should be simplified
+#pragma warning disable S2219 // Runtime type checking should be simplified.
         if (model is KnownModels.AzureAIFoundryEvaluation)
 #pragma warning restore S2219
         {
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/TimingHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/TimingHelper.cs
index 74b68b25b9e..fdd16aef3e2 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/TimingHelper.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/TimingHelper.cs
@@ -3,12 +3,16 @@
 
 using System;
 using System.Diagnostics;
+using System.Globalization;
 using System.Threading.Tasks;
 
 namespace Microsoft.Extensions.AI.Evaluation.Utilities;
 
 internal static class TimingHelper
 {
+    internal static string ToMillisecondsText(this TimeSpan duration) =>
+        duration.TotalMilliseconds.ToString("F2", CultureInfo.InvariantCulture);
+
     internal static TimeSpan ExecuteWithTiming(Action operation)
     {
         Stopwatch stopwatch = Stopwatch.StartNew();
@@ -42,7 +46,7 @@ internal static (TResult result, TimeSpan duration) ExecuteWithTiming<TResult>(F
         return (result, duration: stopwatch.Elapsed);
     }
 
-#pragma warning disable EA0014 // The async method doesn't support cancellation
+#pragma warning disable EA0014 // The async method doesn't support cancellation.
     internal static async ValueTask<TimeSpan> ExecuteWithTimingAsync(Func<Task> operation)
     {
         Stopwatch stopwatch = Stopwatch.StartNew();
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/NLPEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/NLPEvaluatorTests.cs
index a4f3b75045a..9cd593a647a 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/NLPEvaluatorTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/NLPEvaluatorTests.cs
@@ -1,9 +1,6 @@
 ﻿// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#pragma warning disable CA2016 // Forward the 'CancellationToken' parameter to methods that take it.
-#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor.
-
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs
index c867fca4926..fde342a4161 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs
@@ -1,9 +1,6 @@
 ﻿// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#pragma warning disable CA2016 // Forward the 'CancellationToken' parameter to methods that take it.
-#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor.
-
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
@@ -116,7 +113,7 @@ public async Task SampleMultipleResponses()
         SkipIfNotConfigured();
 
 #if NET
-        await Parallel.ForAsync(1, 6, async (i, _) =>
+        await Parallel.ForAsync(1, 6, async (i, cancellationToken) =>
 #else
         for (int i = 1; i < 6; i++)
 #endif
@@ -124,7 +121,8 @@ await Parallel.ForAsync(1, 6, async (i, _) =>
             await using ScenarioRun scenarioRun =
                 await _qualityReportingConfiguration.CreateScenarioRunAsync(
                     scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(QualityEvaluatorTests)}.{nameof(SampleMultipleResponses)}",
-                    iterationName: i.ToString());
+                    iterationName: i.ToString(),
+                    cancellationToken: cancellationToken);
 
             IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient;
 
@@ -132,9 +130,10 @@ await _qualityReportingConfiguration.CreateScenarioRunAsync(
             string prompt = @"How far in miles is the planet Venus from the Earth at its closest and furthest points?";
             messages.Add(prompt.ToUserMessage());
 
-            ChatResponse response = await chatClient.GetResponseAsync(messages, _chatOptions);
+            ChatResponse response = await chatClient.GetResponseAsync(messages, _chatOptions, cancellationToken);
 
-            EvaluationResult result = await scenarioRun.EvaluateAsync(messages, response);
+            EvaluationResult result =
+                await scenarioRun.EvaluateAsync(messages, response, cancellationToken: cancellationToken);
 
             Assert.False(
                 result.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Warning),
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
index c2c02103714..0e5ba2eed1b 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
@@ -1,8 +1,6 @@
 ﻿// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
-
 using System;
 using System.Collections.Generic;
 using System.IO;
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/BLEUEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/BLEUEvaluatorTests.cs
index ae651f30c73..64f304d2da3 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/BLEUEvaluatorTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/BLEUEvaluatorTests.cs
@@ -1,13 +1,14 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+#pragma warning disable AIEVAL001
+// AIEVAL001: Some of the below APIs are experimental and subject to change or removal in future updates.
+
 using System.Threading.Tasks;
 using Xunit;
 
 namespace Microsoft.Extensions.AI.Evaluation.NLP.Tests;
 
-#pragma warning disable AIEVAL001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-
 public class BLEUEvaluatorTests
 {
     [Fact]
@@ -107,5 +108,4 @@ public async Task ReturnsErrorDiagnosticIfEmptyResponse()
         Assert.NotNull(metric.Diagnostics);
         Assert.Contains(metric.Diagnostics, d => d.Severity == EvaluationDiagnosticSeverity.Error);
     }
-
 }
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/F1EvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/F1EvaluatorTests.cs
index 6e71e252329..23dfc6963a0 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/F1EvaluatorTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/F1EvaluatorTests.cs
@@ -1,13 +1,14 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+#pragma warning disable AIEVAL001
+// AIEVAL001: Some of the below APIs are experimental and subject to change or removal in future updates.
+
 using System.Threading.Tasks;
 using Xunit;
 
 namespace Microsoft.Extensions.AI.Evaluation.NLP.Tests;
 
-#pragma warning disable AIEVAL001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-
 public class F1EvaluatorTests
 {
     [Fact]
@@ -87,5 +88,4 @@ public async Task ReturnsErrorDiagnosticIfEmptyResponse()
         Assert.NotNull(metric.Diagnostics);
         Assert.Contains(metric.Diagnostics, d => d.Severity == EvaluationDiagnosticSeverity.Error);
     }
-
 }
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/GLEUEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/GLEUEvaluatorTests.cs
index 56f63c17d34..df007fcc638 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/GLEUEvaluatorTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/GLEUEvaluatorTests.cs
@@ -1,13 +1,14 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+#pragma warning disable AIEVAL001
+// AIEVAL001: Some of the below APIs are experimental and subject to change or removal in future updates.
+
 using System.Threading.Tasks;
 using Xunit;
 
 namespace Microsoft.Extensions.AI.Evaluation.NLP.Tests;
 
-#pragma warning disable AIEVAL001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-
 public class GLEUEvaluatorTests
 {
     [Fact]
@@ -107,5 +108,4 @@ public async Task ReturnsErrorDiagnosticIfEmptyResponse()
         Assert.NotNull(metric.Diagnostics);
         Assert.Contains(metric.Diagnostics, d => d.Severity == EvaluationDiagnosticSeverity.Error);
     }
-
 }
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/SimpleTokenizerTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/SimpleTokenizerTests.cs
index 5766c2e7fc0..2906bf1ae5e 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/SimpleTokenizerTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.NLP.Tests/SimpleTokenizerTests.cs
@@ -6,8 +6,6 @@
 
 namespace Microsoft.Extensions.AI.Evaluation.NLP.Tests;
 
-#pragma warning disable AIEVAL001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
-
 public class SimpleTokenizerTests
 {
     [Theory]
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs
index 1f74f66136c..50793dfdb66 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs
@@ -2,9 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
-#pragma warning disable S1128 // Unused "using" should be removed
-using System.Linq;
-#pragma warning restore S1128
 using System.Text;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Caching.Distributed;
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
index 84324183091..637678f6be2 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
@@ -391,7 +391,7 @@ private class ChatTurnDetailsComparer : IEqualityComparer<ChatTurnDetails>
     {
         public static ChatTurnDetailsComparer Instance { get; } = new ChatTurnDetailsComparer();
 
-#pragma warning disable S1067 // Expressions should not be too complex
+#pragma warning disable S1067 // Expressions should not be too complex.
         public bool Equals(ChatTurnDetails? x, ChatTurnDetails? y) =>
             x?.Latency == y?.Latency &&
             x?.Model == y?.Model &&
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/SerializationChainingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/SerializationChainingTests.cs
index 86a319c059d..bebc0d5affd 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/SerializationChainingTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/SerializationChainingTests.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
-using System.Collections.Generic;
 using System.Text.Json;
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization;
@@ -24,17 +23,14 @@ public class SerializationChainingTests
             messages: [],
             modelResponse: new ChatResponse
             {
-                Messages = new List<ChatMessage>
-                {
+                Messages =
+                [
                     new ChatMessage
                     {
                         Role = ChatRole.User,
-                        Contents = new List<AIContent>
-                        {
-                            new TextContent("A user message"),
-                        },
+                        Contents = [new TextContent("A user message")]
                     },
-                },
+                ],
                 AdditionalProperties = new AdditionalPropertiesDictionary
                 {
                     { "model", "gpt-7" },
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs
similarity index 78%
rename from test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs
rename to test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs
index 12267943a2e..555f72c1f15 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs
@@ -8,7 +8,7 @@
 
 namespace Microsoft.Extensions.AI.Evaluation.Tests;
 
-public class BuiltInEvaluatorUtilitiesTests
+public class BuiltInMetricUtilitiesTests
 {
     [Fact]
     public void MarkAsBuiltInAddsMetadata()
@@ -32,7 +32,7 @@ public void IsBuiltInReturnsFalseIfMetadataIsMissing()
     public void MetadataValueOfTrueIsCaseInsensitive(string value)
     {
         var metric = new BooleanMetric("name");
-        metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, value);
+        metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, value);
         Assert.True(metric.IsBuiltIn());
     }
 
@@ -43,7 +43,7 @@ public void MetadataValueOfTrueIsCaseInsensitive(string value)
     public void MetadataValueOfFalseIsCaseInsensitive(string value)
     {
         var metric = new StringMetric("name");
-        metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, value);
+        metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, value);
         Assert.False(metric.IsBuiltIn());
     }
 
@@ -51,7 +51,7 @@ public void MetadataValueOfFalseIsCaseInsensitive(string value)
     public void UnrecognizedMetadataValueIsTreatedAsFalse()
     {
         var metric = new NumericMetric("name");
-        metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, "unrecognized");
+        metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, "unrecognized");
         Assert.False(metric.IsBuiltIn());
     }
 
@@ -59,7 +59,7 @@ public void UnrecognizedMetadataValueIsTreatedAsFalse()
     public void EmptyMetadataValueIsTreatedAsFalse()
     {
         var metric = new NumericMetric("name");
-        metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, string.Empty);
+        metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, string.Empty);
         Assert.False(metric.IsBuiltIn());
     }
 }
