Skip to content

Commit

Permalink
Refactor TestMethodRunner.Execute so that it returns TestResult (from…
Browse files Browse the repository at this point in the history 
… TestFramework) rather than UnitTestResult (from TestAdapter) (#4591)
  • Loading branch information
@Youssef1313
Youssef1313 authored Jan 13, 2025
1 parent 1cedd92 commit c749676
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 39 deletions.
41 changes: 21 additions & 20 deletions src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,15 @@ public TestMethodRunner(TestMethodInfo testMethodInfo, TestMethod testMethod, IT
/// Executes a test.
/// </summary>
/// <returns>The test results.</returns>
internal UnitTestResult[] Execute(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
internal List<TestResult> Execute(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
{
bool isSTATestClass = AttributeComparer.IsDerived<STATestClassAttribute>(_testMethodInfo.Parent.ClassAttribute);
bool isSTATestMethod = AttributeComparer.IsDerived<STATestMethodAttribute>(_testMethodInfo.TestMethodOptions.Executor);
bool isSTARequested = isSTATestClass || isSTATestMethod;
bool isWindowsOS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
if (isSTARequested && isWindowsOS && Thread.CurrentThread.GetApartmentState() != ApartmentState.STA)
{
UnitTestResult[] results = Array.Empty<UnitTestResult>();
List<TestResult>? results = null;
Thread entryPointThread = new(() => results = SafeRunTestMethod(initializationLogs, initializationErrorLogs, initializationTrace, initializationTestContextMessages))
{
Name = (isSTATestClass, isSTATestMethod) switch
Expand All @@ -93,7 +93,7 @@ internal UnitTestResult[] Execute(string initializationLogs, string initializati
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(ex.ToString());
}

return results;
return results ?? new();
}
else
{
Expand All @@ -107,42 +107,43 @@ internal UnitTestResult[] Execute(string initializationLogs, string initializati
}

// Local functions
UnitTestResult[] SafeRunTestMethod(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
List<TestResult> SafeRunTestMethod(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
{
UnitTestResult[]? result = null;
List<TestResult>? result = null;

try
{
result = RunTestMethod();
}
catch (TestFailedException ex)
{
result = [new UnitTestResult(ex)];
result = [new TestResult() { TestFailureException = ex }];
}
catch (Exception ex)
{
if (result == null || result.Length == 0)
if (result == null || result.Count == 0)
{
result = [new()];
result = [new TestResult() { Outcome = UTF.UnitTestOutcome.Error }];
}

#pragma warning disable IDE0056 // Use index operator
result[result.Length - 1] = new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()))
result[result.Count - 1] = new TestResult()
{
StandardOut = result[result.Length - 1].StandardOut,
StandardError = result[result.Length - 1].StandardError,
DebugTrace = result[result.Length - 1].DebugTrace,
TestContextMessages = result[result.Length - 1].TestContextMessages,
Duration = result[result.Length - 1].Duration,
TestFailureException = new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()),
LogOutput = result[result.Count - 1].LogOutput,
LogError = result[result.Count - 1].LogError,
DebugTrace = result[result.Count - 1].DebugTrace,
TestContextMessages = result[result.Count - 1].TestContextMessages,
Duration = result[result.Count - 1].Duration,
};
#pragma warning restore IDE0056 // Use index operator
}
finally
{
// Assembly initialize and class initialize logs are pre-pended to the first result.
UnitTestResult firstResult = result![0];
firstResult.StandardOut = initializationLogs + firstResult.StandardOut;
firstResult.StandardError = initializationErrorLogs + firstResult.StandardError;
TestResult firstResult = result![0];
firstResult.LogOutput = initializationLogs + firstResult.LogOutput;
firstResult.LogError = initializationErrorLogs + firstResult.LogError;
firstResult.DebugTrace = initializationTrace + firstResult.DebugTrace;
firstResult.TestContextMessages = initializationTestContextMessages + firstResult.TestContextMessages;
}
Expand All @@ -155,7 +156,7 @@ UnitTestResult[] SafeRunTestMethod(string initializationLogs, string initializat
/// Runs the test method.
/// </summary>
/// <returns>The test results.</returns>
internal UnitTestResult[] RunTestMethod()
internal List<TestResult> RunTestMethod()
{
DebugEx.Assert(_test != null, "Test should not be null.");
DebugEx.Assert(_testMethodInfo.TestMethod != null, "Test method should not be null.");
Expand All @@ -172,7 +173,7 @@ internal UnitTestResult[] RunTestMethod()
{
if (_test.TestDataSourceIgnoreMessage is not null)
{
return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreMessage)];
return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }];
}

object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
Expand Down Expand Up @@ -243,7 +244,7 @@ internal UnitTestResult[] RunTestMethod()
results.Add(emptyResult);
}

return results.ToUnitTestResults();
return results;
}

private bool TryExecuteDataSourceBasedTests(List<TestResult> results)
Expand Down
2 changes: 1 addition & 1 deletion src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<strin
// Run the test method
testContextForTestExecution.SetOutcome(testContextForClassInit.Context.CurrentTestOutcome);
var testMethodRunner = new TestMethodRunner(testMethodInfo, testMethod, testContextForTestExecution);
result = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!);
result = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!).ToUnitTestResults();
}
}
}
Expand Down
37 changes: 19 additions & 18 deletions test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodRunnerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices;
Expand Down Expand Up @@ -82,7 +83,7 @@ public void ExecuteForTestThrowingExceptionShouldReturnUnitTestResultWithFailedO
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => throw new Exception("DummyException"));
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Verify(results[0].ErrorMessage.Contains("Exception thrown while executing test"));
}
Expand All @@ -92,7 +93,7 @@ public void ExecuteForPassingTestShouldReturnUnitTestResultWithPassedOutcome()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
}

Expand All @@ -104,7 +105,7 @@ public void ExecuteShouldNotFillInDebugAndTraceLogsIfDebugTraceDisabled()
StringWriter writer = new(new StringBuilder("DummyTrace"));
_testablePlatformServiceProvider.MockTraceListener.Setup(tl => tl.GetWriter()).Returns(writer);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].DebugTrace == string.Empty);
}

Expand All @@ -126,7 +127,7 @@ public void ExecuteShouldNotFillInDebugAndTraceLogsFromRunningTestMethod()
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);
_testablePlatformServiceProvider.MockTraceListener.Setup(tl => tl.GetWriter()).Returns(writer);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();

Verify(results[0].DebugTrace == string.Empty);
}
Expand All @@ -136,7 +137,7 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => throw new Exception("Dummy Exception"));
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Verify(results[0].ErrorMessage.Contains("Exception thrown while executing test"));
}
Expand All @@ -155,7 +156,7 @@ public void RunTestMethodForMultipleResultsReturnMultipleResults()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, localTestMethodOptions, null);
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results.Length == 2);

Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand All @@ -167,7 +168,7 @@ public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResu
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
}

Expand All @@ -176,7 +177,7 @@ public void RunTestMethodForFailingTestThrowingExceptionShouldReturnUnitTestResu
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Failed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
}

Expand All @@ -185,7 +186,7 @@ public void RunTestMethodShouldGiveTestResultAsPassedWhenTestMethodPasses()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// Since data is not provided, tests run normally giving passed as outcome.
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand All @@ -196,7 +197,7 @@ public void RunTestMethodShouldGiveTestResultAsFailedWhenTestMethodFails()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Failed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// Since data is not provided, tests run normally giving passed as outcome.
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Expand All @@ -217,7 +218,7 @@ public void RunTestMethodShouldRunDataDrivenTestsWhenDataIsProvidedUsingDataSour
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for outcome
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand Down Expand Up @@ -246,7 +247,7 @@ public void RunTestMethodShouldRunDataDrivenTestsWhenDataIsProvidedUsingDataRowA
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(_methodInfo, It.IsAny<Type>(), It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Inconclusive);
}

Expand All @@ -263,7 +264,7 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for datarowIndex
Verify(results[0].DatarowIndex == 0);
Expand All @@ -289,7 +290,7 @@ public void RunTestMethodShouldRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for datarowIndex as only DataSource Tests are Run
Verify(results[0].DatarowIndex == 0);
Expand All @@ -315,7 +316,7 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

Verify(results.Length == 1);
Verify(results[0].DisplayName == "DataRowTestDisplayName");
Expand All @@ -338,7 +339,7 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

Verify(results.Length == 1);
Verify(results[0].DisplayName is "dummyTestName (2,\"DummyString\")" or "DummyTestMethod (2,DummyString)", $"Display name: {results[0].DisplayName}");
Expand All @@ -364,7 +365,7 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests()
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].ResultFiles.ToList().Contains("C:\\temp.txt"));
Verify(results[1].ResultFiles.ToList().Contains("C:\\temp.txt"));
}
Expand Down Expand Up @@ -404,7 +405,7 @@ private void RunTestMethodWithEmptyDataSourceShouldFailIfConsiderEmptyDataSource

if (considerEmptyAsInconclusive)
{
UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Inconclusive);
}
else
Expand Down

0 comments on commit c749676

Please sign in to comment.