From bcc1e90cb840ae3053c93c75784e89f17ab54ab8 Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 11:58:29 +0200 Subject: [PATCH 1/8] add Repeat helper --- tests/FSharp.Test.Utilities/XunitHelpers.fs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index cf7ecf3b2cf..1ab4a8aa65e 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -21,6 +21,14 @@ open OpenTelemetry.Trace [] type RunTestCasesInSequenceAttribute() = inherit Attribute() +// Helper for stress testing. +// Runs a test case many times in parallel. +// Example usage: [] +type RepeatAttribute([] data: obj array) = + inherit DataAttribute() + member val Count = 1 with get, set + override this.GetData _ = Seq.init this.Count (fun i -> [| yield! data; yield box i |]) + #if XUNIT_EXTRAS // To use xUnit means to customize it. The following abomination adds 2 features: From bc7dfc84490c924d30c070a5ae2d3598c0ef9ce7 Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 11:58:43 +0200 Subject: [PATCH 2/8] fix flaky test --- .../CompilerService/AsyncMemoize.fs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs index f37b64eb0f5..bd7a162afd8 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs @@ -422,18 +422,19 @@ let ``Cancel running jobs with the same key`` () = // detach requests from their running computations cts.Cancel() + // Cancel the Get reequests, leaving the jobs running unawaited. for job in jobsToCancel do assertTaskCanceled job + // Start another request. let job = cache.Get(key 11, work) |> Async.StartAsTask // up til now the jobs should have been running unobserved let current = eventsWhen events (received Requested) Assert.Equal(0, current |> countOf Canceled) - // waitUntil events (countOf Canceled >> (=) 10) - - waitUntil events (received Started) + waitUntil events (countOf Started >> (=) 11) + // Allow the single current request to finish. jobCanContinue.Set() |> ignore job.Wait() From be15f41ba36e39230c93f42c1a2a745f5443e97b Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:14:34 +0200 Subject: [PATCH 3/8] Update tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs Co-authored-by: Petr --- .../CompilerService/AsyncMemoize.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs index bd7a162afd8..77e8f4b2137 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs @@ -422,7 +422,7 @@ let ``Cancel running jobs with the same key`` () = // detach requests from their running computations cts.Cancel() - // Cancel the Get reequests, leaving the jobs running unawaited. + // Cancel the Get requests, leaving the jobs running unawaited. for job in jobsToCancel do assertTaskCanceled job // Start another request. From 87f62812910b9cd98f79404b3ef50edb7aa60e37 Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:41:57 +0200 Subject: [PATCH 4/8] update docs --- DEVGUIDE.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 28b511fac04..d96f3994a56 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -252,6 +252,15 @@ For example: module TimeCritical = ``` +For stress testing async code you can use a custom `FSharp.Test.RepeatAttribute`. +For example, applied to a single xUnit test case: +```fsharp +[] +``` +it will start it many times at the same time, and execute in parallel. + + + ### Updating FCS surface area baselines From 41be29d4de3d00949f731cad3e62c51992ea027c Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 16:02:50 +0200 Subject: [PATCH 5/8] naming --- DEVGUIDE.md | 4 ++-- tests/FSharp.Test.Utilities/XunitHelpers.fs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index d96f3994a56..c551c90af2b 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -252,10 +252,10 @@ For example: module TimeCritical = ``` -For stress testing async code you can use a custom `FSharp.Test.RepeatAttribute`. +For stress testing async code you can use a custom `FSharp.Test.StressAttribute`. For example, applied to a single xUnit test case: ```fsharp -[] +[] ``` it will start it many times at the same time, and execute in parallel. diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index 1ab4a8aa65e..34a44df17ed 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -23,8 +23,8 @@ type RunTestCasesInSequenceAttribute() = inherit Attribute() // Helper for stress testing. // Runs a test case many times in parallel. -// Example usage: [] -type RepeatAttribute([] data: obj array) = +// Example usage: [] +type StressAttribute([] data: obj array) = inherit DataAttribute() member val Count = 1 with get, set override this.GetData _ = Seq.init this.Count (fun i -> [| yield! data; yield box i |]) From 668afd34d3973baf4b36c2c3f6492b8659d0c21d Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Mon, 7 Apr 2025 20:33:57 +0200 Subject: [PATCH 6/8] add tests --- .../FSharp.Test.Utilities.fsproj | 20 +++++--------- tests/FSharp.Test.Utilities/Tests.fs | 26 +++++++++++++++++++ tests/FSharp.Test.Utilities/xunit.runner.json | 5 ++++ 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 tests/FSharp.Test.Utilities/Tests.fs create mode 100644 tests/FSharp.Test.Utilities/xunit.runner.json diff --git a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj index 37c1940b7a9..b98f57603c6 100644 --- a/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj +++ b/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj @@ -10,7 +10,8 @@ true false false - false + xunit + true $(OtherFlags) --warnon:1182 --realsig- true @@ -23,10 +24,9 @@ - - - - + + PreserveNewest + scriptlib.fsx @@ -46,11 +46,10 @@ + + - - - @@ -70,11 +69,6 @@ - - - - - diff --git a/tests/FSharp.Test.Utilities/Tests.fs b/tests/FSharp.Test.Utilities/Tests.fs new file mode 100644 index 00000000000..4ef630286e1 --- /dev/null +++ b/tests/FSharp.Test.Utilities/Tests.fs @@ -0,0 +1,26 @@ +module FSharp.Test.StressAttributeTests + +open System.Threading +open Xunit +open FSharp.Test + +type RunOrFail(name) = + let mutable count = 0 + member _.Run(shouldFail: bool) = + let count = Interlocked.Increment &count + if shouldFail && count = 42 then + failwith $"{name}, failed as expected on {count}" + else + printfn $"{name}, iteration {count} passed" + count + +let passing = RunOrFail "Passing" +let failing = RunOrFail "Failing" + +[] +let ``Stress attribute should catch intermittent failure`` shouldFail _ = + failing.Run shouldFail + +[] +let ``Stress attribute works`` _ = + passing.Run false \ No newline at end of file diff --git a/tests/FSharp.Test.Utilities/xunit.runner.json b/tests/FSharp.Test.Utilities/xunit.runner.json new file mode 100644 index 00000000000..b01c50a3cb5 --- /dev/null +++ b/tests/FSharp.Test.Utilities/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "appDomain": "denied", + "parallelizeAssembly": true +} From 060042fc3c8e3d8cc35372a9a2361fcc5083d24d Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Tue, 8 Apr 2025 23:50:08 +0200 Subject: [PATCH 7/8] tests --- tests/FSharp.Test.Utilities/Tests.fs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/FSharp.Test.Utilities/Tests.fs b/tests/FSharp.Test.Utilities/Tests.fs index 4ef630286e1..58d407cda64 100644 --- a/tests/FSharp.Test.Utilities/Tests.fs +++ b/tests/FSharp.Test.Utilities/Tests.fs @@ -1,5 +1,6 @@ -module FSharp.Test.StressAttributeTests +module FSharp.Test.UtilitiesTests +open System open System.Threading open Xunit open FSharp.Test @@ -23,4 +24,22 @@ let ``Stress attribute should catch intermittent failure`` shouldFail _ = [] let ``Stress attribute works`` _ = - passing.Run false \ No newline at end of file + passing.Run false + +[] +let ``TestConsole captures output`` () = + let rnd = Random() + let task n = + async { + use console = new TestConsole.ExecutionCapture() + do! Async.Sleep (rnd.Next 50) + printf $"Hello, world! {n}" + do! Async.Sleep (rnd.Next 50) + eprintf $"Some error {n}" + return console.OutText, console.ErrorText + } + + let expected = [ for n in 0 .. 9 -> $"Hello, world! {n}", $"Some error {n}" ] + + let results = Seq.init 10 task |> Async.Parallel |> Async.RunSynchronously + Assert.Equal(expected, results) From e29ddc1652df83023ecaba1fab3585c86407b649 Mon Sep 17 00:00:00 2001 From: Jakub Majocha <1760221+majocha@users.noreply.github.com> Date: Wed, 9 Apr 2025 18:59:16 +0200 Subject: [PATCH 8/8] run tests in ci --- eng/Build.ps1 | 3 ++- eng/build.sh | 1 + tests/FSharp.Test.Utilities/Tests.fs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 32dcca19336..91ed6cd4008 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -615,7 +615,8 @@ try { if ($testDesktop -and $ci) { $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true - + + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Test.Utilities\" TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" diff --git a/eng/build.sh b/eng/build.sh index 1263445973f..650f78163e5 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -331,6 +331,7 @@ BuildSolution if [[ "$test_core_clr" == true ]]; then coreclrtestframework=$tfm + Test --testproject "$repo_root/tests/FSharp.Test.Utilities/FSharp.Test.Utilities.fsproj" --targetframework $coreclrtestframework Test --testproject "$repo_root/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" --targetframework $coreclrtestframework Test --testproject "$repo_root/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj" --targetframework $coreclrtestframework Test --testproject "$repo_root/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj" --targetframework $coreclrtestframework diff --git a/tests/FSharp.Test.Utilities/Tests.fs b/tests/FSharp.Test.Utilities/Tests.fs index 58d407cda64..9561f1bf08f 100644 --- a/tests/FSharp.Test.Utilities/Tests.fs +++ b/tests/FSharp.Test.Utilities/Tests.fs @@ -22,7 +22,7 @@ let failing = RunOrFail "Failing" let ``Stress attribute should catch intermittent failure`` shouldFail _ = failing.Run shouldFail -[] +[] let ``Stress attribute works`` _ = passing.Run false