Skip to content

Apply structured assertion messages (RFC 012) to Assert.Fail / Assert.Inconclusive #8296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Evangelink wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Apply structured assertion messages (RFC 012) to Assert.Fail / Assert.Inconclusive #8296

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67fa1bb
Apply structured assertion messages (RFC 012) to Assert.Fail / Assert…
Copilot May 16, 2026
e24a4c5
Fix review feedback for fail and inconclusive assertions
Copilot May 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/TestFramework/TestFramework/Assertions/Assert.Fail.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ public sealed partial class Assert
public static void Fail(string message = "")
{
TelemetryCollector.TrackAssertionCall("Assert.Fail");
ThrowAssertFailed("Assert.Fail", BuildUserMessage(message));

StructuredAssertionMessage structured = new(string.Empty);
structured.WithUserMessage(message);
ThrowAssertFailed(structured);
}
}
10 changes: 8 additions & 2 deletions src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@ public static void Inconclusive(string message = "")
TelemetryCollector.TrackAssertionCall("Assert.Inconclusive");

string userMessage = BuildUserMessage(message);
throw new AssertInconclusiveException(
FormatAssertionFailed("Assert.Inconclusive", userMessage));
throw new AssertInconclusiveException(FormatInconclusive(userMessage));
}

private static string FormatInconclusive(string? message)
=> string.IsNullOrWhiteSpace(message)
? FrameworkMessages.InconclusivePrefix
: message![0] is '\n' or '\r'
? string.Concat(FrameworkMessages.InconclusivePrefix, message)
: $"{FrameworkMessages.InconclusivePrefix} {message}";
}
3 changes: 3 additions & 0 deletions src/TestFramework/TestFramework/Resources/FrameworkMessages.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,9 @@ Actual: {2}</value>
<data name="AssertionFailed" xml:space="preserve">
<value>{0} failed.</value>
</data>
<data name="InconclusivePrefix" xml:space="preserve">
<value>Assert.Inconclusive.</value>
</data>
<data name="IsInstanceOfFailMsg" xml:space="preserve">
<value>{0} Expected type:&lt;{1}&gt;. Actual type:&lt;{2}&gt;.</value>
</data>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.cs.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Skutečnost: {2}</target>
<target state="translated">Očekávala se kolekce {1} velikosti. Skutečnost: {2} {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">Vlastnost TestContext.{0} souvisí s aktuálním testem a není k dispozici během sestavení nebo používání testovacích přípravků tříd.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.de.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Tatsächlich: {2}</target>
<target state="translated">Es wurde eine Sammlung mit einer Größe {1} erwartet. Tatsächlich: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">Die Eigenschaft „TestContext.{0}“ im Zusammenhang mit dem aktuellen Test steht während Assembly- oder Klassenfixierungen nicht zur Verfügung.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.es.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Real: {2}</target>
<target state="translated">Se esperaba una colección de tamaño {1}. Real: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">La propiedad "TestContext.{0}" está relacionado con la prueba actual y no está disponible durante los accesorios de ensamblado o clase.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.fr.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Réel : {2}</target>
<target state="translated">Collection de tailles attendue {1}. Réel : {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">La propriété « TestContext.{0} », liée au test en cours, n’est pas disponible pendant les fixtures d’assembly ou de classe.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.it.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Effettivo: {2}</target>
<target state="translated">Prevista raccolta di dimensioni {1}. Effettivo: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">La proprietà 'TestContext.{0}' relativa al test corrente non è disponibile durante le fixture di assembly o classe.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.ja.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Actual: {2}</source>
<target state="translated">サイズ {1} のコレクションが必要です。実際: {2}。{0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">プロパティ 'TestContext.{0}' は現在のテストに関連しており、アセンブリまたはクラス フィクスチャの間は使用できません。</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.ko.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Actual: {2}</source>
<target state="translated">{1} 크기 컬렉션이 필요합니다. 실제: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">현재 테스트와 관련된 'TestContext.{0}' 속성은 어셈블리나 클래스 픽스처 실행 중에는 사용할 수 없습니다.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.pl.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Rzeczywiste: {2}</target>
<target state="translated">Oczekiwano kolekcji rozmiaru {1}. Wartość rzeczywista: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">Właściwość „TestContext.{0}” jest powiązana z bieżącym testem i nie jest dostępna podczas montażu lub konfiguracji klasy.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.pt-BR.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Real: {2}</target>
<target state="translated">Coleção esperada de tamanho {1}. Real: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">A propriedade "TestContext.{0}" está relacionada ao teste atual não está disponível durante os acessórios de assembly ou classe.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.ru.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Actual: {2}</source>
<target state="translated">Ожидается коллекция размеров {1}. Фактически: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">Свойство "TestContext.{0}" связано с текущим тестом и недоступно во время выполнения средств тестирования сборок или классов.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.tr.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Gerçekte olan: {2}</target>
<target state="translated">Beklenen boyut {1}. Gerçek: {2}. {0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">'TestContext.{0}' özelliği, derleme veya sınıf sabitlemeleri sırasında mevcut testle ilişkili değildir.</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.zh-Hans.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Actual: {2}</source>
<target state="translated">大小 {1} 的预期集合。实际： {2}。{0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">属性‘TestContext.{0}’与当前测试相关，在程序集或类固定例程期间不可用。</target>
Expand Down
5 changes: 5 additions & 0 deletions src/TestFramework/TestFramework/Resources/xlf/FrameworkMessages.zh-Hant.xlf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,11 @@ Actual: {2}</source>
<target state="translated">預期的大小集合 {1}。實際： {2}。{0}</target>
<note />
</trans-unit>
<trans-unit id="InconclusivePrefix">
<source>Assert.Inconclusive.</source>
<target state="new">Assert.Inconclusive.</target>
<note />
</trans-unit>
<trans-unit id="InvalidAccessToTestContextProperty">
<source>The property 'TestContext.{0}' is related to current test is not available during assembly or class fixtures.</source>
<target state="translated">屬性 'TestContext.{0}' 與目前測試相關，無法在組件或類別測試夾具期間使用。</target>
Expand Down
2 changes: 1 addition & 1 deletion test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
""");

testHostResult.AssertOutputContains("failed TestMethod5");
testHostResult.AssertOutputContains("Assert.Fail failed. Failing TestMethod4. Attempts: 4 (from TestContext: 4)");
testHostResult.AssertOutputContains("Assertion failed.\r\nFailing TestMethod4. Attempts: 4 (from TestContext: 4)");

Check failure on line 29 in test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs

View check run for this annotation

Azure Pipelines / microsoft.testfx (Build Linux Debug)

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs#L29 

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs(29,1): error : [BasicRetryScenarioTest] [net11.0] Assert.Contains failed. String 'MSTest v4.3.0-ci (UTC 05/16/2026) [linux-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /mnt/vss/_work/1/s/artifacts/tmp/Debug/testsuite/TpfFu/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /mnt/vss/_work/1/s/artifacts/tmp/Debug/testsuite/TpfFu/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 396ms' does not contain string 'Assertion failed.
Failing TestMethod4. Attempts: 4 (from TestContext: 4)'. 'substring' expression: 'value', 'value' expression: 'testHostResult.StandardOutput'. Expression 'AssertOutputContains' failed for member 'BasicRetryScenarioTest' at line 29 of file '/_/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs'. Output of the test host is:
Command: /mnt/vss/_work/1/s/artifacts/tmp/Debug/testsuite/TpfFu/RetryTests/bin/Release/net10.0/RetryTests --settings my.runsettings
====================
ExitCode: 2
====================
StandardOutput:
MSTest v4.3.0-ci (UTC 05/16/2026) [linux-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /mnt/vss/_work/1/s/artifacts/tmp/Debug/testsuite/TpfFu/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /mnt/vss/_work/1/s/artifacts/tmp/Debug/testsuite/TpfFu/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 396ms
====================
StandardError:
.

Check failure on line 29 in test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs

View check run for this annotation

Azure Pipelines / microsoft.testfx (Build MacOS Debug)

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs#L29 

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs(29,1): error : [BasicRetryScenarioTest] [net11.0] Assert.Contains failed. String 'MSTest v4.3.0-ci (UTC 5/16/2026) [osx-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 1s 841ms' does not contain string 'Assertion failed.
Failing TestMethod4. Attempts: 4 (from TestContext: 4)'. 'substring' expression: 'value', 'value' expression: 'testHostResult.StandardOutput'. Expression 'AssertOutputContains' failed for member 'BasicRetryScenarioTest' at line 29 of file '/_/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs'. Output of the test host is:
Command: /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests --settings my.runsettings
====================
ExitCode: 2
====================
StandardOutput:
MSTest v4.3.0-ci (UTC 5/16/2026) [osx-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 1s 841ms
====================
StandardError:
.

Check failure on line 29 in test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs

View check run for this annotation

Azure Pipelines / microsoft.testfx (Build Windows Debug)

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs#L29 

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs(29,1): error : [BasicRetryScenarioTest] [net11.0] Assert.Contains failed. String 'MSTest v4.3.0-ci (UTC 5/16/2026) [win-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in D:\a\_work\1\s\artifacts\tmp\Debug\testsuite\mpEtN\RetryTests\UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - D:\a\_work\1\s\artifacts\tmp\Debug\testsuite\mpEtN\RetryTests\bin\Release\net10.0\RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 405ms' does not contain string 'Assertion failed.
Failing TestMethod4. Attempts: 4 (from TestContext: 4)'. 'substring' expression: 'value', 'value' expression: 'testHostResult.StandardOutput'. Expression 'AssertOutputContains' failed for member 'BasicRetryScenarioTest' at line 29 of file '/_/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs'. Output of the test host is:
Command: D:\a\_work\1\s\artifacts\tmp\Debug\testsuite\mpEtN\RetryTests\bin\Release\net10.0\RetryTests.exe --settings my.runsettings
====================
ExitCode: 2
====================
StandardOutput:
MSTest v4.3.0-ci (UTC 5/16/2026) [win-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in D:\a\_work\1\s\artifacts\tmp\Debug\testsuite\mpEtN\RetryTests\UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - D:\a\_work\1\s\artifacts\tmp\Debug\testsuite\mpEtN\RetryTests\bin\Release\net10.0\RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 405ms
====================
StandardError:
.

Check failure on line 29 in test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs

View check run for this annotation

Azure Pipelines / microsoft.testfx

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs#L29 

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs(29,1): error : [BasicRetryScenarioTest] [net11.0] Assert.Contains failed. String 'MSTest v4.3.0-ci (UTC 5/16/2026) [osx-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 1s 841ms' does not contain string 'Assertion failed.
Failing TestMethod4. Attempts: 4 (from TestContext: 4)'. 'substring' expression: 'value', 'value' expression: 'testHostResult.StandardOutput'. Expression 'AssertOutputContains' failed for member 'BasicRetryScenarioTest' at line 29 of file '/_/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RetryTests.cs'. Output of the test host is:
Command: /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests --settings my.runsettings
====================
ExitCode: 2
====================
StandardOutput:
MSTest v4.3.0-ci (UTC 5/16/2026) [osx-x64 - .NET 10.0.8]
TestMethod1 executed 1 time.
TestMethod2 executed 2 times.
TestMethod3 executed 3 times.
TestMethod4 executed 4 times.
TestMethod5 executed 4 times.
failed TestMethod5 (0ms)
  Assertion failed.
  Failing TestMethod4. Attempts: 4 (from TestContext: 4)
    at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(String message) in /_/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs:30
    at UnitTest1.TestMethod5() in /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/UnitTest1.cs:57
    at InvokeStub_UnitTest1.TestMethod5(Object, Object, IntPtr*)
    at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
Test run summary: Failed! - /Users/runner/work/1/s/artifacts/tmp/Debug/testsuite/fVt4U/RetryTests/bin/Release/net10.0/RetryTests.dll (net10.0|x64)
  total: 5
  failed: 1
  succeeded: 4
  skipped: 0
  duration: 1s 841ms
====================
StandardError:
.
testHostResult.AssertOutputContainsSummary(failed: 1, passed: 4, skipped: 0);
}

Expand Down
8 changes: 4 additions & 4 deletions test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestAssemblyInfoTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public async Task RunAssemblyInitializeShouldThrowTestFailedExceptionOnAssertion
var exception = (await _testAssemblyInfo.RunAssemblyInitializeAsync(_testContext)).TestFailureException as TestFailedException;
exception.Should().NotBeNull();
exception.Outcome.Should().Be(UnitTestOutcome.Failed);
exception.Message.Should().Be("Assembly Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests+DummyTestClass.AssemblyInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException: Assert.Fail failed. Test failure. Aborting test execution.");
exception.Message.Should().Be("Assembly Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests+DummyTestClass.AssemblyInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException: Assertion failed.\r\nTest failure. Aborting test execution.");
exception.StackTraceInformation!.ErrorStackTrace.Should().Contain(
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests.DummyTestClass.AssemblyInitializeMethod");
exception.InnerException.Should().BeOfType<AssertFailedException>();
Expand All @@ -162,7 +162,7 @@ public async Task RunAssemblyInitializeShouldThrowTestFailedExceptionWithInconcl
var exception = (await _testAssemblyInfo.RunAssemblyInitializeAsync(_testContext)).TestFailureException as TestFailedException;
exception.Should().NotBeNull();
exception.Outcome.Should().Be(UnitTestOutcome.Inconclusive);
exception.Message.Should().Be("Assembly Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests+DummyTestClass.AssemblyInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertInconclusiveException: Assert.Inconclusive failed. Test Inconclusive. Aborting test execution.");
exception.Message.Should().Be("Assembly Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests+DummyTestClass.AssemblyInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertInconclusiveException: Assert.Inconclusive. Test Inconclusive. Aborting test execution.");
exception.StackTraceInformation!.ErrorStackTrace.Should().Contain(
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestAssemblyInfoTests.DummyTestClass.AssemblyInitializeMethod");
exception.InnerException.Should().BeOfType<AssertInconclusiveException>();
Expand Down Expand Up @@ -264,7 +264,7 @@ public async Task RunAssemblyCleanupShouldReturnAssertFailureExceptionDetails()
_testAssemblyInfo.AssemblyCleanupMethod = typeof(DummyTestClass).GetMethod("AssemblyCleanupMethod")!;
string? actualErrorMessage = (await _testAssemblyInfo.ExecuteAssemblyCleanupAsync(GetTestContext()))?.Message;
actualErrorMessage!.StartsWith(
"Assembly Cleanup method DummyTestClass.AssemblyCleanupMethod failed. Error Message: Assert.Fail failed. Test Failure..", StringComparison.Ordinal).Should().BeTrue($"Value: {actualErrorMessage}");
"Assembly Cleanup method DummyTestClass.AssemblyCleanupMethod failed. Error Message: Assertion failed.\r\nTest Failure..", StringComparison.Ordinal).Should().BeTrue($"Value: {actualErrorMessage}");
}

public async Task RunAssemblyCleanupShouldReturnAssertInconclusiveExceptionDetails()
Expand All @@ -276,7 +276,7 @@ public async Task RunAssemblyCleanupShouldReturnAssertInconclusiveExceptionDetai
_testAssemblyInfo.AssemblyCleanupMethod = typeof(DummyTestClass).GetMethod("AssemblyCleanupMethod")!;
string? actualErrorMessage = (await _testAssemblyInfo.ExecuteAssemblyCleanupAsync(GetTestContext()))?.Message;
actualErrorMessage!.StartsWith(
"Assembly Cleanup method DummyTestClass.AssemblyCleanupMethod failed. Error Message: Assert.Inconclusive failed. Test Inconclusive..", StringComparison.Ordinal).Should().BeTrue($"Value: {actualErrorMessage}");
"Assembly Cleanup method DummyTestClass.AssemblyCleanupMethod failed. Error Message: Assert.Inconclusive. Test Inconclusive..", StringComparison.Ordinal).Should().BeTrue($"Value: {actualErrorMessage}");
}

public async Task RunAssemblyCleanupShouldReturnExceptionDetailsOfNonAssertExceptions()
Expand Down
8 changes: 4 additions & 4 deletions test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestClassInfoTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ public void RunClassInitializeShouldThrowTestFailedExceptionOnAssertionFailure()

exception.Should().NotBeNull();
exception.Outcome.Should().Be(UnitTestOutcome.Failed);
exception.Message.Should().Be("Class Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests+DummyTestClass.ClassInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException: Assert.Fail failed. Test failure.");
exception.Message.Should().Be("Class Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests+DummyTestClass.ClassInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertFailedException: Assertion failed.\r\nTest failure.");
exception.StackTraceInformation!.ErrorStackTrace.Contains(
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests.DummyTestClass.ClassInitializeMethod", StringComparison.Ordinal).Should().BeTrue();
exception.InnerException.Should().BeOfType<AssertFailedException>();
Expand All @@ -367,7 +367,7 @@ public void RunClassInitializeShouldThrowTestFailedExceptionWithInconclusiveOnAs

exception.Should().NotBeNull();
exception.Outcome.Should().Be(UnitTestOutcome.Inconclusive);
exception.Message.Should().Be("Class Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests+DummyTestClass.ClassInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertInconclusiveException: Assert.Inconclusive failed. Test Inconclusive.");
exception.Message.Should().Be("Class Initialization method Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests+DummyTestClass.ClassInitializeMethod threw exception. Microsoft.VisualStudio.TestTools.UnitTesting.AssertInconclusiveException: Assert.Inconclusive. Test Inconclusive.");
exception.StackTraceInformation!.ErrorStackTrace.Contains(
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestClassInfoTests.DummyTestClass.ClassInitializeMethod", StringComparison.Ordinal).Should().BeTrue();
exception.InnerException.Should().BeOfType<AssertInconclusiveException>();
Expand Down Expand Up @@ -481,7 +481,7 @@ public async Task RunClassCleanupShouldReturnAssertFailureExceptionDetails()
// Assert
classCleanupException.Should().NotBeNull();
classCleanupException.Message.StartsWith("Class Cleanup method DummyTestClass.ClassCleanupMethod failed.", StringComparison.Ordinal).Should().BeTrue();
classCleanupException.Message.Contains("Error Message: Assert.Fail failed. Test Failure.").Should().BeTrue();
classCleanupException.Message.Contains("Error Message: Assertion failed.\r\nTest Failure.").Should().BeTrue();
classCleanupException.Message.Should().Contain(
$"{typeof(TestClassInfoTests).FullName}.DummyTestClass.ClassCleanupMethod",
$"Value: {classCleanupException.Message}");
Expand All @@ -502,7 +502,7 @@ public async Task RunClassCleanupShouldReturnAssertInconclusiveExceptionDetails(
// Assert
classCleanupException.Should().NotBeNull();
classCleanupException.Message.StartsWith("Class Cleanup method DummyTestClass.ClassCleanupMethod failed.", StringComparison.Ordinal).Should().BeTrue();
classCleanupException.Message.Contains("Error Message: Assert.Inconclusive failed. Test Inconclusive.").Should().BeTrue();
classCleanupException.Message.Contains("Error Message: Assert.Inconclusive. Test Inconclusive.").Should().BeTrue();
classCleanupException.Message.Should().Contain(
$"{typeof(TestClassInfoTests).FullName}.DummyTestClass.ClassCleanupMethod",
$"Value: {classCleanupException.Message}");
Expand Down
Loading
Loading