chore: Update test projects to use xunit v3

.github/workflows/generate-coverage-report.yaml

@@ -44,13 +44,13 @@ jobs:
 
       - name: Collect Code Coverage
         run: |
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net8.0"
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net8.0"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net8.0"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net8.0"
 
       - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests 
         if: ${{ github.event.inputs.skip_integration_tests == 'false'}} 
         run: |
-          dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net8.0 --filter "(FullyQualifiedName!~DotMemoryTests) & (FullyQualifiedName!~DotTraceTests) & (FullyQualifiedName!~WasmIsSupported) & (FullyQualifiedName!~WasmSupportsInProcessDiagnosers)"'
+          dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net8.0 --filter-query "/*/*/(!*DotMemoryTests)&(!*DotTraceTests)&(!*WasmTests)"' --no-ansi --output Detailed
 
       - name: Shutdown dotnet-coverage server.
         run: dotnet coverage shutdown bdn_coverage --timeout 60000
@@ -80,13 +80,13 @@ jobs:
 
       - name: Collect Code Coverage
         run: |
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472"
-          dotnet coverage connect bdn_coverage "dotnet test tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net472"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Tests -c Release --no-build --framework net472"
+          dotnet coverage connect bdn_coverage "dotnet test --project tests/BenchmarkDotNet.Analyzers.Tests  -c Release --no-build --framework net472"
 
       - name: Collect Code Coverage for BenchmarkDotNet.IntegrationTests 
         if: ${{ github.event.inputs.skip_integration_tests == 'false'}} 
         run: |
-          dotnet coverage connect bdn_coverage 'dotnet test tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --filter "(FullyQualifiedName!~DotMemoryTests) & (FullyQualifiedName!~DotTraceTests) & (FullyQualifiedName!~WasmIsSupported) & (FullyQualifiedName!~WasmSupportsInProcessDiagnosers)"'
+          dotnet coverage connect bdn_coverage 'dotnet test --project tests/BenchmarkDotNet.IntegrationTests -c Release --no-build --framework net472 --filter-query "/*/*/(!*DotMemoryTests)&(!*DotTraceTests)&(!*WasmTests)"' --no-ansi --output Detailed
 
       - name: Shutdown dotnet-coverage server.
         run: dotnet coverage shutdown bdn_coverage --timeout 60000

.github/workflows/run-tests-selected.yaml

@@ -34,7 +34,7 @@ on:
           - net472
       filter:
         type: string
-        description: Test filter text (It's used for `dotnet test --filter`) Use default value when running all tests
+        description: Test filter text (It's used for `dotnet test --filter-method`) Use default value when running all tests
         required: true
         default: "BenchmarkDotNet"
       iteration_count:
@@ -91,12 +91,13 @@ jobs:
         run: |
           $PSNativeCommandUseErrorActionPreference = $true
           $iterationCount = ${{ inputs.iteration_count }}
+
+          Write-Host ("OSArchitecture" + [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)
 
           foreach($i in 1..$iterationCount) {
             Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i)
 
-            dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal"
-
+            dotnet test -c Release --framework ${{ inputs.framework }} --no-build --filter-method "${{ inputs.filter }}" --no-ansi
             Write-Output '##[endgroup]'
           }
 

.github/workflows/run-tests.yaml

@@ -63,7 +63,9 @@ jobs:
         if: always()
         with:
           name: test-windows-core-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-windows-full:
     strategy:
@@ -116,7 +118,9 @@ jobs:
         if: always()
         with:
           name: test-windows-full-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-linux:
     strategy:
@@ -164,7 +168,9 @@ jobs:
         if: always()
         with:
           name: test-linux-trx-${{ github.run_id }}-${{ matrix.os }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-macos:
     name: test-macos (${{ matrix.os.arch }})
@@ -211,7 +217,9 @@ jobs:
         if: always()
         with:
           name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }}
-          path: "**/*.trx"
+          path: |
+            **/*.trx
+            **/log_*.diag
 
   test-pack:
     runs-on: ubuntu-latest

BenchmarkDotNet.slnx

@@ -28,7 +28,9 @@
     <Project Path="templates/BenchmarkDotNet.Templates.csproj" />
   </Folder>
   <Folder Name="/tests/">
+    <File Path="tests/.editorconfig" />
     <File Path="tests/Directory.Build.props" />
+    <File Path="tests/Directory.Build.targets" />
     <Project Path="tests/BenchmarkDotNet.Analyzers.Tests/BenchmarkDotNet.Analyzers.Tests.csproj" />
     <Project Path="tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj" />
     <Project Path="tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj" />

build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj

@@ -7,7 +7,7 @@
     <NoWarn>$(NoWarn);CA2007</NoWarn>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Cake.Frosting" Version="6.0.0" />
+    <PackageReference Include="Cake.Frosting" Version="6.1.0" />
     <PackageReference Include="Cake.FileHelpers" Version="7.0.0" />
     <PackageReference Include="Cake.Git" Version="5.0.1" />
     <PackageReference Include="Docfx.App" Version="2.78.5" />

build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs

@@ -3,7 +3,9 @@
 using Cake.Common.Diagnostics;
 using Cake.Common.Tools.DotNet;
 using Cake.Common.Tools.DotNet.Test;
+using Cake.Core;
 using Cake.Core.IO;
+using System.Linq;
 using System.Runtime.InteropServices;
 
 namespace BenchmarkDotNet.Build.Runners;
@@ -41,11 +43,18 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf
             Framework = tfm,
             NoBuild = true,
             NoRestore = true,
-            Loggers = new[] { "trx", $"trx;LogFileName={logFile.FullPath}", "console;verbosity=detailed" },
             EnvironmentVariables =
             {
                 ["Platform"] = "" // force the tool to not look for the .dll in platform-specific directory
-            }
+            },
+            PathType = DotNetTestPathType.Auto,
+            ArgumentCustomization = args
+                => args.Append("--report-xunit-trx")
+                    .AppendSwitchQuoted("--report-xunit-trx-filename", System.IO.Path.GetFileName(logFile.FullPath))
+                    .Append("--no-ansi")
+                    .AppendSwitch("--output", "Detailed")
+                    .Append("--diagnostic")
+                    .AppendSwitch("--diagnostic-verbosity", "Trace")
         };
         return settings;
     }

global.json

@@ -0,0 +1,5 @@
+{
+  "test": {
+    "runner": "Microsoft.Testing.Platform"
+  }
+}

src/BenchmarkDotNet/Templates/CsProj.txt

@@ -27,6 +27,7 @@
     <!-- Suppress warning for nuget package used in old (unsupported) tfm. -->
     <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
     <StartupObject>BenchmarkDotNet.Autogenerated.UniqueProgramName</StartupObject>
+    <IsTestingPlatformApplication>false</IsTestingPlatformApplication>
   </PropertyGroup>
 
   <ItemGroup>

src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk" DefaultTarget="Publish">
+<Project Sdk="Microsoft.NET.Sdk" DefaultTarget="Publish">
   <PropertyGroup>
     <OriginalCSProjPath>$CSPROJPATH$</OriginalCSProjPath>
     <MonoPropsPath>$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props'))</MonoPropsPath>
@@ -21,6 +21,7 @@
     <SelfContained>true</SelfContained>
     <!-- Suppress warning for nuget package used in old (unsupported) tfm. -->
     <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
+    <IsTestingPlatformApplication>false</IsTestingPlatformApplication>
   </PropertyGroup>
 
   <ItemGroup>

src/BenchmarkDotNet/Templates/R2RCsProj.txt

@@ -30,6 +30,7 @@
     <StartupObject>BenchmarkDotNet.Autogenerated.UniqueProgramName</StartupObject>
     <!-- workaround for 'Found multiple publish output files with the same relative path.' error -->
     <ErrorOnDuplicatePublishOutputFiles>false</ErrorOnDuplicatePublishOutputFiles>
+    <IsTestingPlatformApplication>false</IsTestingPlatformApplication>
   </PropertyGroup>
 
   <PropertyGroup>

src/BenchmarkDotNet/Templates/WasmCsProj.txt

@@ -1,4 +1,4 @@
-<Project Sdk="$SDKNAME$" DefaultTargets="publish">
+<Project Sdk="$SDKNAME$" DefaultTargets="publish">
   <PropertyGroup>
     <OriginalCSProjPath>$CSPROJPATH$</OriginalCSProjPath>
     <WasmPropsPath>$([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Wasm.props'))</WasmPropsPath>
@@ -26,6 +26,7 @@ $CORECLR_OVERRIDES$
     <!-- Suppress warning for nuget package used in old (unsupported) tfm. -->
     <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
     <StartupObject>BenchmarkDotNet.Autogenerated.UniqueProgramName</StartupObject>
+    <IsTestingPlatformApplication>false</IsTestingPlatformApplication>
   </PropertyGroup>
 
   <ItemGroup>

src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs

@@ -158,6 +158,10 @@ private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartiti
                 <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings> <!-- Suppress warning for nuget package used in old (unsupported) tfm. -->
                 {GetInstructionSetSettings(buildPartition)}
               </PropertyGroup>
+              <PropertyGroup>
+                <IsTestingPlatformApplication>false</IsTestingPlatformApplication>
+                <XunitAutoGeneratedEntryPoint>false</XunitAutoGeneratedEntryPoint>
+              </PropertyGroup>
               {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)}
               <ItemGroup>
                 <Compile Include="{Path.GetFileName(artifactsPaths.ProgramCodePath)}" Exclude="bin\**;obj\**;**\*.xproj;packages\**" />

tests/.editorconfig

@@ -0,0 +1,7 @@
+root = false
+
+# C# files
+[*.cs]
+# xUnit1051: Calls to methods which accept CancellationToken should use TestContext.Current.CancellationToken to allow test cancellation to be more responsive.
+dotnet_diagnostic.xUnit1051.severity = suggestion
+

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/ArgumentsAttributeAnalyzerTests.cs

@@ -1110,7 +1110,7 @@ public void BenchmarkMethod({{integerValueAndType.Value2}} a)
         }
 
         public static IEnumerable<string> DummyAttributeUsage
-            => DummyAttributeUsageTheoryData;
+            => DummyAttributeUsageTheoryData.Select(x => x.Data);
 
         public static TheoryData<string> EmptyArgumentsAttributeUsagesWithMismatchingValueCount()
         {

tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/Attributes/GeneralParameterAttributesAnalyzerTests.cs

@@ -5,6 +5,7 @@
 using Xunit;
 
 namespace BenchmarkDotNet.Analyzers.Tests.AnalyzerTests.Attributes;
+
 public class GeneralParameterAttributesAnalyzerTests
 {
     public class MutuallyExclusiveOnField : AnalyzerTestFixture<GeneralParameterAttributesAnalyzer>
@@ -158,7 +159,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, int, int[]> DuplicateSameParameterAttributeUsages
             => DuplicateSameAttributeUsagesTheoryData;
@@ -320,7 +321,7 @@ public class BenchmarkClass
             await RunAsync();
         }
 
-        public static TheoryData<string> UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+        public static TheoryData<string> UniqueParameterAttributeUsages => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, int, int[]> DuplicateSameParameterAttributeUsages => DuplicateSameAttributeUsagesTheoryData;
 
@@ -479,21 +480,19 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicCla
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers);
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<string> NonPublicClassMemberAccessModifiers
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new NonPublicClassMemberAccessModifiersTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data);
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int) tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3));
 
         public static IEnumerable<int[]> DuplicateParameterAttributeUsageCounts
-            => DuplicateAttributeUsageCountsTheoryData;
+            => DuplicateAttributeUsageCountsTheoryData.Select(x => x.Data);
     }
 
     public class PropertyMustBePublic : AnalyzerTestFixture<GeneralParameterAttributesAnalyzer>
@@ -648,18 +647,16 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicCla
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicClassMemberAccessModifiers);
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<string> NonPublicClassMemberAccessModifiers
-#pragma warning disable IDE0028 // Simplify collection initialization
-            => new NonPublicClassMemberAccessModifiersTheoryData();
-#pragma warning restore IDE0028 // Simplify collection initialization
+            => new NonPublicClassMemberAccessModifiersTheoryData().Select(x => x.Data);
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int) tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3));
 
         public static TheoryData<int[]> DuplicateParameterAttributeUsageCounts
             => DuplicateAttributeUsageCountsTheoryData;
@@ -813,7 +810,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, string> UniqueParameterAttributes
             => UniqueParameterAttributesTheoryData;
@@ -951,7 +948,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, string> UniqueParameterAttributes
             => UniqueParameterAttributesTheoryData;
@@ -1113,7 +1110,7 @@ public class BenchmarkClass
         }
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static TheoryData<string, string> UniqueParameterAttributes
             => UniqueParameterAttributesTheoryData;
@@ -1278,20 +1275,20 @@ public static IEnumerable<object[]> DuplicateAttributeUsageCountsAndNonPublicPro
             => CombinationsGenerator.CombineArguments(DuplicateParameterAttributeUsageCounts, NonPublicPropertySetters());
 
         public static TheoryData<string> UniqueParameterAttributeUsages
-            => [.. UniqueParameterAttributesTheoryData.Select(tdr => (tdr[1] as string)!)];
+            => [.. UniqueParameterAttributesTheoryData.Select(tdr => tdr.Data.Item2)];
 
         public static IEnumerable<(string AttributeName, string AttributeUsage)> UniqueParameterAttributes
-            => UniqueParameterAttributesTheoryData.Select(tdr => ((tdr[0] as string)!, (tdr[1] as string)!));
+            => UniqueParameterAttributesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2));
 
         public static IEnumerable<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> DuplicateSameParameterAttributeUsages
-            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => ((tdr[0] as string)!, (int) tdr[1], (tdr[2] as int[])!));
+            => DuplicateSameAttributeUsagesTheoryData.Select(tdr => (tdr.Data.Item1, tdr.Data.Item2, tdr.Data.Item3));
 
         public static TheoryData<int[]> DuplicateParameterAttributeUsageCounts
             => DuplicateAttributeUsageCountsTheoryData;
 
         public static IEnumerable<string> NonPublicPropertySetters()
             => new NonPublicPropertySetterAccessModifiersTheoryData()
-            .Select<string, string>(m => $"{{ get; {m} set; }}")
+            .Select(m => $"{{ get; {m.Data} set; }}")
             .Concat(["{ get; }", "=> 0;"]);
 
         public static TheoryData<string> NonPublicPropertySettersTheoryData()
@@ -1441,7 +1438,7 @@ public void Run() { }
             await RunAsync();
         }
     }
-    
+
     public static TheoryData<string, string> UniqueParameterAttributesTheoryData
         => new()
         {
@@ -1486,7 +1483,7 @@ private static IEnumerable<int[]> GenerateDuplicateAttributeUsageCombinations(Th
     private static ReadOnlyCollection<(string CurrentUniqueAttributeUsage, int CurrentUniqueAttributeUsagePosition, int[] Counts)> GenerateDuplicateSameAttributeUsageCombinations(TheoryData<string, string> uniqueAttributeUsages)
     {
         var uniqueAttributeUsagesList = uniqueAttributeUsages
-            .Select(tdr => (tdr[1] as string)!)
+            .Select(tdr => tdr.Data.Item2)
             .ToList()
             .AsReadOnly();