From 482ec639e894352719d432a34c7e4abfff9c6c99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hanno=20J=2E=20G=C3=B6decke?= <die.drei99@yahoo.de>
Date: Thu, 15 Feb 2024 09:10:01 +0100
Subject: [PATCH] fail e2e tests after three errors

---
 tests/e2e/testRunner.js | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js
index 63c88d207581..feb440440f4f 100644
--- a/tests/e2e/testRunner.js
+++ b/tests/e2e/testRunner.js
@@ -324,6 +324,11 @@ const runTests = async () => {
 
         // We run each test multiple time to average out the results
         const testLog = Logger.progressInfo('');
+        // For each test case we allow the test to fail three times before we stop the test run:
+        const errorCountRef = {
+            errorCount: 0,
+            allowedExceptions: 3,
+        };
         for (let i = 0; i < config.RUNS; i++) {
             progressText = `Suite '${suite.name}' [${suiteIndex + 1}/${suites.length}], iteration [${i + 1}/${config.RUNS}]\n`;
             testLog.updateText(progressText);
@@ -341,9 +346,11 @@ const runTests = async () => {
 
             const onError = (e) => {
                 testLog.done();
-                if (i === 0) {
+                errorCountRef.errorCount += 1;
+                if (i === 0 || errorCountRef.errorCount === errorCountRef.allowedExceptions) {
                     // If the error happened on the first test run, the test is broken
-                    // and we should not continue running it
+                    // and we should not continue running it. Or if we have reached the
+                    // maximum number of allowed exceptions, we should stop the test run.
                     throw e;
                 }
                 console.error(e);