From f51024ea31256379ff9bcffb8c7b160f0387b500 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Bignon?= <cbignon@microsoft.com>
Date: Tue, 6 Feb 2018 10:02:01 +0100
Subject: [PATCH 1/4] Fix Multiple results in VS2017 test runner (#336)

---
 .../Execution/TestMethodRunner.cs             | 37 ++++++----
 test/E2ETests/Automation.CLI/CLITestBase.cs   |  1 +
 .../CustomTestExecutionExtensibilityTests.cs  | 32 +++++++++
 .../Smoke.E2E.Tests/Smoke.E2E.Tests.csproj    |  1 +
 .../CustomTestExTests.cs                      | 69 +++++++++++++++++++
 .../FxExtensibilityTestProject.csproj         |  1 +
 .../Execution/TestMethodRunnerTests.cs        | 29 +++++++-
 7 files changed, 157 insertions(+), 13 deletions(-)
 create mode 100644 test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
 create mode 100644 test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs

diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
index 5559f91338..3e7b45b14b 100644
--- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
+++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
@@ -247,22 +247,28 @@ internal UnitTestResult[] RunTestMethod()
                                     watch.Start();
 
                                     this.testContext.SetDataRow(dataRow);
-                                    UTF.TestResult currentResult;
+                                    UTF.TestResult[] testResults;
 
                                     try
                                     {
-                                        currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+                                        testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
                                     }
                                     catch (Exception ex)
                                     {
-                                        currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+                                        testResults = new[]
+                                        {
+                                            new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+                                        };
                                     }
 
-                                    currentResult.DatarowIndex = rowIndex++;
                                     watch.Stop();
-                                    currentResult.Duration = watch.Elapsed;
+                                    foreach (var testResult in testResults)
+                                    {
+                                        testResult.DatarowIndex = rowIndex++;
+                                        testResult.Duration = watch.Elapsed;
+                                    }
 
-                                    results.Add(currentResult);
+                                    results.AddRange(testResults);
                                 }
                             }
                             finally
@@ -293,18 +299,25 @@ internal UnitTestResult[] RunTestMethod()
                             foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo))
                             {
                                 this.testMethodInfo.SetArguments(data);
-                                UTF.TestResult currentResult;
+                                UTF.TestResult[] testResults;
                                 try
                                 {
-                                    currentResult = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0];
+                                    testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo);
                                 }
                                 catch (Exception ex)
                                 {
-                                    currentResult = new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) };
+                                    testResults = new[]
+                                    {
+                                        new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }
+                                    };
+                                }
+
+                                foreach (var testResult in testResults)
+                                {
+                                    testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
                                 }
 
-                                currentResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data);
-                                results.Add(currentResult);
+                                results.AddRange(testResults);
                                 this.testMethodInfo.SetArguments(null);
                             }
                         }
@@ -313,7 +326,7 @@ internal UnitTestResult[] RunTestMethod()
                     {
                         try
                         {
-                            results.Add(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)[0]);
+                            results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo));
                         }
                         catch (Exception ex)
                         {
diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs
index 88cd82f668..d09f0d6a64 100644
--- a/test/E2ETests/Automation.CLI/CLITestBase.cs
+++ b/test/E2ETests/Automation.CLI/CLITestBase.cs
@@ -160,6 +160,7 @@ public void ValidatePassedTestsContain(params string[] passedTests)
         {
             foreach (var test in passedTests)
             {
+                var x = string.Join(", ", this.runEventsHandler.PassedTests.Select(_ => _.DisplayName));
                 var testFound = this.runEventsHandler.PassedTests.Any(
                     p => test.Equals(p.TestCase?.FullyQualifiedName)
                          || test.Equals(p.DisplayName));
diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
new file mode 100644
index 0000000000..e8b6115cec
--- /dev/null
+++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
@@ -0,0 +1,32 @@
+﻿// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace MSTestAdapter.Smoke.E2ETests
+{
+    using Microsoft.MSTestV2.CLIAutomation;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class CustomTestExecutionExtensibilityTests : CLITestBase
+    {
+        private const string TestAssembly = "FxExtensibilityTestProject.dll";
+
+        [TestMethod]
+        public void ExecuteCustomTestExtensibilityTests()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly });
+            this.ValidatePassedTestsContain(
+                "CustomTestMethod1 - Execution number 1",
+                "CustomTestMethod1 - Execution number 2",
+                "CustomTestMethod1 - Execution number 4",
+                "CustomTestMethod1 - Execution number 5",
+                "CustomTestClass1 - Execution number 1",
+                "CustomTestClass1 - Execution number 2",
+                "CustomTestClass1 - Execution number 4",
+                "CustomTestClass1 - Execution number 5");
+            this.ValidateFailedTestsContain(
+                "CustomTestMethod1 - Execution number 3",
+                "CustomTestClass1 - Execution number 3");
+        }
+    }
+}
\ No newline at end of file
diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
index 6005ad1c50..8b694c0e14 100644
--- a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
+++ b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
@@ -39,6 +39,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AssertExtensibilityTests.cs" />
+    <Compile Include="CustomTestExecutionExtensibilityTests.cs" />
     <Compile Include="DataSourceTests.cs" />
     <Compile Include="DesktopCSharpCLITests.cs" />
     <Compile Include="ParallelExecutionTests.cs" />
diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
new file mode 100644
index 0000000000..71732d55a3
--- /dev/null
+++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
@@ -0,0 +1,69 @@
+﻿namespace FxExtensibilityTestProject
+{
+    using System;
+    using System.Collections.Generic;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [IterativeTestClass(5)]
+    public class CustomTestExTests
+    {
+        private static int customTestMethod1ExecutionCount;
+        [IterativeTestMethod(5)]
+        public void CustomTestMethod1()
+        {
+            customTestMethod1ExecutionCount++;
+            Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
+        }
+
+        private static int customTestClass1ExecutionCount;
+        [TestMethod]
+        public void CustomTestClass1()
+        {
+            customTestClass1ExecutionCount++;
+            Assert.AreNotEqual(3, customTestClass1ExecutionCount);
+        }
+    }
+
+    public class IterativeTestMethodAttribute : TestMethodAttribute
+    {
+        private readonly int stabilityThreshold;
+
+        public IterativeTestMethodAttribute(int stabilityThreshold)
+        {
+            this.stabilityThreshold = stabilityThreshold;
+        }
+
+        public override TestResult[] Execute(ITestMethod testMethod)
+        {
+            var results = new List<TestResult>();
+            for (int count = 0; count < this.stabilityThreshold; count++)
+            {
+                var testResults = base.Execute(testMethod);
+                foreach (var testResult in testResults)
+                {
+                    testResult.DisplayName = $"{testMethod.TestMethodName} - Execution number {count + 1}";
+                }
+                results.AddRange(testResults);
+            }
+
+            return results.ToArray();
+        }
+    }
+
+    public class IterativeTestClassAttribute : TestClassAttribute
+    {
+        private readonly int stabilityThreshold;
+
+        public IterativeTestClassAttribute(int stabilityThreshold)
+        {
+            this.stabilityThreshold = stabilityThreshold;
+        }
+
+        public override TestMethodAttribute GetTestMethodAttribute(TestMethodAttribute testMethodAttribute)
+        {
+            if (testMethodAttribute is IterativeTestMethodAttribute) return testMethodAttribute;
+
+            return new IterativeTestMethodAttribute(this.stabilityThreshold);
+        }
+    }
+}
diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
index db9dc23cba..9e587e801d 100644
--- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
+++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/FxExtensibilityTestProject.csproj
@@ -44,6 +44,7 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="DynamicDataExTests.cs" />
     <Compile Include="DynamicDataExMoreTests.cs" />
+    <Compile Include="CustomTestExTests.cs" />
     <Compile Include="TestDataSourceExTests.cs" />
   </ItemGroup>
   <ItemGroup>
diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
index 9f66bad592..3dbd30d23e 100644
--- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
+++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs
@@ -407,10 +407,37 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
             StringAssert.Contains(results[0].ErrorMessage, "Exception thrown while executing test");
         }
 
+        [TestMethodV1]
+        public void RunTestMethodForMultipleResultsReturnMultipleResults()
+        {
+            var testMethodAttributeMock = new Mock<UTF.TestMethodAttribute>();
+            testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny<UTF.ITestMethod>())).Returns(new[]
+            {
+                new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed },
+                new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed }
+            });
+
+            var localTestMethodOptions = new TestMethodOptions
+            {
+                Timeout = 200,
+                Executor = testMethodAttributeMock.Object,
+                TestContext = this.testContextImplementation,
+                ExpectedException = null
+            };
+
+            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null);
+            var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
+
+            var results = testMethodRunner.Execute();
+            Assert.AreEqual(2, results.Length);
+            Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome);
+            Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome);
+        }
+
         [TestMethodV1]
         public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResultWithPassedOutcome()
         {
-            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo,  this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
+            var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
             var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false);
 
             var results = testMethodRunner.Execute();

From 0b48abb9a581ac25236a47ccf3b9659911ffdbee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Bignon?= <cbignon@microsoft.com>
Date: Wed, 7 Feb 2018 10:20:50 +0100
Subject: [PATCH 2/4] Fixed PR comments

---
 src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs | 3 ++-
 test/E2ETests/Automation.CLI/CLITestBase.cs                  | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
index 3e7b45b14b..1ab3be4dd8 100644
--- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
+++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
@@ -261,10 +261,11 @@ internal UnitTestResult[] RunTestMethod()
                                         };
                                     }
 
+                                    rowIndex++;
                                     watch.Stop();
                                     foreach (var testResult in testResults)
                                     {
-                                        testResult.DatarowIndex = rowIndex++;
+                                        testResult.DatarowIndex = rowIndex;
                                         testResult.Duration = watch.Elapsed;
                                     }
 
diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs
index d09f0d6a64..88cd82f668 100644
--- a/test/E2ETests/Automation.CLI/CLITestBase.cs
+++ b/test/E2ETests/Automation.CLI/CLITestBase.cs
@@ -160,7 +160,6 @@ public void ValidatePassedTestsContain(params string[] passedTests)
         {
             foreach (var test in passedTests)
             {
-                var x = string.Join(", ", this.runEventsHandler.PassedTests.Select(_ => _.DisplayName));
                 var testFound = this.runEventsHandler.PassedTests.Any(
                     p => test.Equals(p.TestCase?.FullyQualifiedName)
                          || test.Equals(p.DisplayName));

From d80a0268c1d4f3ea0269820431ede10c26c14a5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Bignon?= <cbignon@microsoft.com>
Date: Wed, 7 Feb 2018 10:21:34 +0100
Subject: [PATCH 3/4] Added tests with data-driven tests

---
 test/E2ETests/Automation.CLI/CLITestBase.cs   |  6 ++++--
 .../CustomTestExecutionExtensibilityTests.cs  | 19 +++++++++++++++++++
 .../CustomTestExTests.cs                      | 10 +++++++++-
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs
index 88cd82f668..f846b3e69a 100644
--- a/test/E2ETests/Automation.CLI/CLITestBase.cs
+++ b/test/E2ETests/Automation.CLI/CLITestBase.cs
@@ -10,6 +10,7 @@ namespace Microsoft.MSTestV2.CLIAutomation
     using System.Linq;
     using System.Xml;
     using Microsoft.TestPlatform.VsTestConsole.TranslationLayer;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
     public class CLITestBase
@@ -59,7 +60,8 @@ public void InvokeVsTestForDiscovery(string[] sources, string runSettings = "")
         /// </summary>
         /// <param name="sources">List of test assemblies.</param>
         /// <param name="runSettings">Run settings for execution.</param>
-        public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
+        /// <param name="testCaseFilter">Test Case filter for execution.</param>
+        public void InvokeVsTestForExecution(string[] sources, string runSettings = "", string testCaseFilter = null)
         {
             for (var iterator = 0; iterator < sources.Length; iterator++)
             {
@@ -74,7 +76,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "")
 
             // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
             vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
-            vsTestConsoleWrapper.RunTests(sources, runSettingXml, this.runEventsHandler);
+            vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
         }
 
         /// <summary>
diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
index e8b6115cec..23f7cd81c9 100644
--- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
+++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs
@@ -25,8 +25,27 @@ public void ExecuteCustomTestExtensibilityTests()
                 "CustomTestClass1 - Execution number 4",
                 "CustomTestClass1 - Execution number 5");
             this.ValidateFailedTestsContain(
+                TestAssembly,
                 "CustomTestMethod1 - Execution number 3",
                 "CustomTestClass1 - Execution number 3");
         }
+
+        [TestMethod]
+        public void ExecuteCustomTestExtensibilityWithTestDataTests()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2");
+            this.ValidatePassedTests(
+                "CustomTestMethod2 (B)",
+                "CustomTestMethod2 (B)",
+                "CustomTestMethod2 (B)");
+            this.ValidateFailedTests(
+                TestAssembly,
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (A)",
+                "CustomTestMethod2 (C)",
+                "CustomTestMethod2 (C)",
+                "CustomTestMethod2 (C)");
+        }
     }
 }
\ No newline at end of file
diff --git a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
index 71732d55a3..c42be4a431 100644
--- a/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
+++ b/test/E2ETests/TestAssets/FxExtensibilityTestProject/CustomTestExTests.cs
@@ -1,6 +1,5 @@
 ﻿namespace FxExtensibilityTestProject
 {
-    using System;
     using System.Collections.Generic;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -15,6 +14,15 @@ public void CustomTestMethod1()
             Assert.AreNotEqual(3, customTestMethod1ExecutionCount);
         }
 
+        [IterativeTestMethod(3)]
+        [DataRow("A")]
+        [DataRow("B")]
+        [DataRow("C")]
+        public void CustomTestMethod2(string value)
+        {
+            Assert.AreEqual("B", value);
+        }
+
         private static int customTestClass1ExecutionCount;
         [TestMethod]
         public void CustomTestClass1()

From 25bea17ec4d86daccc4db08e477c01b1dc0cee08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Bignon?= <cbignon@microsoft.com>
Date: Wed, 7 Feb 2018 10:53:04 +0100
Subject: [PATCH 4/4] Fixed build

---
 src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
index 1ab3be4dd8..2ece924fcc 100644
--- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
+++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
@@ -261,7 +261,6 @@ internal UnitTestResult[] RunTestMethod()
                                         };
                                     }
 
-                                    rowIndex++;
                                     watch.Stop();
                                     foreach (var testResult in testResults)
                                     {
@@ -269,6 +268,8 @@ internal UnitTestResult[] RunTestMethod()
                                         testResult.Duration = watch.Elapsed;
                                     }
 
+                                    rowIndex++;
+
                                     results.AddRange(testResults);
                                 }
                             }
