https://stryker-mutator.io/assets/js/3fd0caad.9cc76222.js

1"use strict";(self.webpackChunkstryker_mutator_github_io=self.webpackChunkstryker_mutator_github_io||[]).push([["6964"],{19536:function(e,t,s){s.r(t),s.d(t,{metadata:function(){return n},default:function(){return h},frontMatter:function(){return a},contentTitle:function(){return o},toc:function(){return d},assets:function(){return c}});var n=JSON.parse('{"id":"stryker-net/technical-reference/testing-framework","title":"Testing frameworks","description":"This document captures the discoveries made about the various testing frameworks while working on Stryker.","source":"@site/docs/stryker-net/technical-reference/testing-framework.md","sourceDirName":"stryker-net/technical-reference","slug":"/stryker-net/technical-reference/testing-framework","permalink":"/docs/stryker-net/technical-reference/testing-framework","draft":false,"unlisted":false,"editUrl":"https://github.com/stryker-mutator/stryker-net/edit/master/docs/technical-reference/testing-frameworks.md","tags":[],"version":"current","sidebarPosition":50,"frontMatter":{"title":"Testing frameworks","sidebar_position":50,"custom_edit_url":"https://github.com/stryker-mutator/stryker-net/edit/master/docs/technical-reference/testing-frameworks.md"},"sidebar":"docs","previous":{"title":"Project components","permalink":"/docs/stryker-net/technical-reference/project-components"},"next":{"title":"Stryker\u2019s crash course on C# syntax parsing and mutation","permalink":"/docs/stryker-net/technical-reference/Mutation Orchestration Design"}}'),i=s(91987),r=s(67008);let a={title:"Testing frameworks",sidebar_position:50,custom_edit_url:"https://github.com/stryker-mutator/stryker-net/edit/master/docs/technical-reference/testing-frameworks.md"},o="Testing Frameworks undocumented",c={},d=[{value:"Generalities",id:"generalities",level:2},{value:"Identification",id:"identification",level:2},{value:"xUnit",id:"xunit",level:2},{value:"Conflicting test cases",id:"conflicting-test-cases",level:3},{value:"Ignored tests",id:"ignored-tests",level:3},{value:"Theories",id:"theories",level:3},{value:"Static theories",id:"static-theories",level:4},{value:"Run time theories",id:"run-time-theories",level:4},{value:"Run sequence",id:"run-sequence",level:3},{value:"NUnit",id:"nunit",level:2},{value:"Conflicting test cases",id:"conflicting-test-cases-1",level:3},{value:"Ignored tests",id:"ignored-tests-1",level:3},{value:"TestCases with variable data",id:"testcases-with-variable-data",level:3},{value:"TestCase",id:"testcase",level:3},{value:"TestCaseSource",id:"testcasesource",level:3},{value:"Run Sequence",id:"run-sequence-1",level:3}];function l(e){let t={a:"a",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",h4:"h4",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.header,{children:(0,i.jsx)(t.h1,{id:"testing-frameworks-undocumented",children:"Testing Frameworks undocumented"})}),"\n",(0,i.jsx)(t.p,{children:"This document captures the discoveries made about the various testing frameworks while working on Stryker.\nThis can help understanding the design and logic of the mutation test classes.\nThis is a mandatory reading for anyone trying to understand and/or fix bugs related to coverage capture, analysis and testing phase."}),"\n",(0,i.jsx)(t.p,{children:"For the sake of simplicity, this document confuses testing frameworks with their respective VsTest adapters.\nThis is because what matters to Stryker is how the framework/runner behaves when integrated with VsTests."}),"\n",(0,i.jsx)(t.h2,{id:"generalities",children:"Generalities"}),"\n",(0,i.jsxs)(t.p,{children:["The ",(0,i.jsx)(t.a,{href:"https://github.com/Microsoft/vstest-docs/blob/master/RFCs/0006-DataCollection-Protocol.md",children:"VsTest protocol"})," is not\nrich enough to cover every test framework feature. Therefore, each framework makes their own decisions on how to handle\nsome of their specificities. That is the reason why we can observe different behavior and surprising results."]}),"\n",(0,i.jsx)(t.p,{children:"This affects:"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:"how tests are discovered"}),"\n",(0,i.jsx)(t.li,{children:"how tests are described/identified"}),"\n",(0,i.jsx)(t.li,{children:"how tests are filtered"}),"\n",(0,i.jsx)(t.li,{children:"how tests are executed"}),"\n",(0,i.jsx)(t.li,{children:"how test results are reported"}),"\n"]}),"\n",(0,i.jsx)(t.h2,{id:"identification",children:"Identification"}),"\n",(0,i.jsxs)(t.p,{children:["By default, VsTest generates unique identifiers stored as ",(0,i.jsx)(t.code,{children:"Guid"}),". I do not know if test runners are able to provide\na specific implementation but in practice, this identifier is derived from the(test's) display's name hash code."]}),"\n",(0,i.jsx)(t.p,{children:"Note that this is a weaker authentication method than it may appear and there are several situations where several tests (from the coder\u2019s perspective) will appear as a single test case (from VsTest\u2019s point of view).\nTest frameworks adapters can deal with this with various strategies."}),"\n",(0,i.jsx)(t.h2,{id:"xunit",children:"xUnit"}),"\n",(0,i.jsx)(t.h3,{id:"conflicting-test-cases",children:"Conflicting test cases"}),"\n",(0,i.jsxs)(t.p,{children:["If multiple xUnit test cases have the same id (",(0,i.jsx)(t.em,{children:"Guid"}),"), xUnit will report each of those as test results for the same VsTest test case."]}),"\n",(0,i.jsx)(t.h3,{id:"ignored-tests",children:"Ignored tests"}),"\n",(0,i.jsx)(t.p,{children:"Ignored test cases are fully ignored by xUnit runner and are not reported whatsoever."}),"\n",(0,i.jsx)(t.h3,{id:"theories",children:"Theories"}),"\n",(0,i.jsxs)(t.p,{children:["xUnit's theory is a pattern where some test is executed multiple times with varying input data (like NUnit's TestCase).\nThese test methods accept one or more parameters and bear one of this attributes: ",(0,i.jsx)(t.code,{children:"InlineData"}),", ",(0,i.jsx)(t.code,{children:"MemberData"})," or ",(0,i.jsx)(t.code,{children:"ClassData"}),".\nThere are differences\nbetween compile time theories (data is fixed at build time) and run time theories (data is discovered at run time).\nFrom a VsTest perspective, the sub cases of the theory share a single display name."]}),"\n",(0,i.jsx)(t.h4,{id:"static-theories",children:"Static theories"}),"\n",(0,i.jsxs)(t.p,{children:["Static theories are seen and processed as different test cases, unless two (or more) test cases have the same parameters. Static theories used the ",(0,i.jsx)(t.code,{children:"InlineData"})]}),"\n",(0,i.jsx)(t.h4,{id:"run-time-theories",children:"Run time theories"}),"\n",(0,i.jsx)(t.p,{children:"Run time theories may be discovered as one test, disregarding the number of underlying data set.\nDuring discovery phase, xUnit maps each data set to a test case;

1 test case may be different (if they result in a new display name) or the same than an existing ne.\nif it can, those test cases are processed as usual; otherwise it will provide several results for the same test case."}),"\n",(0,i.jsx)(t.p,{children:"It will then run all tests."}),"\n",(0,i.jsx)(t.h3,{id:"run-sequence",children:"Run sequence"}),"\n",(0,i.jsx)(t.p,{children:"Overall, xUnit first discovers every tests at (test) startup. If a list of tests is provided, xUnit filters the discovered lists,\ndiscarding those which are not part of the provided list. Note that there will be no report for requested tests that have not been\nfound during discovery.\nHere is a summarized timeline of tests execution:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{children:"TestSession start event\nxUnit discovers test\n   xUnit discovers theories\n     xUnit enumerates data source (for tests)\n...\nTestCase start event\n   xUnit runs test with first set of data\n   xUnit runs test with second set of data\n   ...\n   xUnit runs test with last set of data\n\n   xUnit reports first test results\nTestCase end event\n   xUnit reports second test result\n   ...\n   xUnit reports last test result\nTestCase start event\n   ...\nTestSession end event\n"})}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.em,{children:"Remark"}),": in this situation, each test result includes the TOTAL execution time (i.e. the time to test every data set) so one should not sum them."]}),"\n",(0,i.jsxs)(t.p,{children:["The difficulty here is that a lot happens between ",(0,i.jsx)(t.code,{children:"testcase end"})," and ",(0,i.jsx)(t.code,{children:"testcase start"})," events.\nAt coverage capture it is a problem, because there is a risk of spilling coverage information to the next tests: Stryker\ncan only capture coverage information on ",(0,i.jsx)(t.code,{children:"testcase end"})," events, and only in association with the current running test.\nSo, every mutants covered between ",(0,i.jsx)(t.code,{children:"testcase end"})," and ",(0,i.jsx)(t.code,{children:"testcase start"})," will be associated with the ",(0,i.jsx)(t.strong,{children:"next"})," test."]}),"\n",(0,i.jsx)(t.p,{children:"And during execution phase, it is impossible to predict when the test will really be over, so it is difficult to\nestablish if the test was successful."}),"\n",(0,i.jsxs)(t.p,{children:["Also, if a mutation ends up changing a test case name - typically by changing the result of ",(0,i.jsx)(t.code,{children:"ToString()"}),", it will change the\ntest identifier so this testcase will only run when running ",(0,i.jsx)(t.strong,{children:"all tests"})," and can no longer be executed in isolation, as\nStryker can't anticipate the test name."]}),"\n",(0,i.jsx)(t.h2,{id:"nunit",children:"NUnit"}),"\n",(0,i.jsx)(t.h3,{id:"conflicting-test-cases-1",children:"Conflicting test cases"}),"\n",(0,i.jsxs)(t.p,{children:["If multiple NUnit test cases have the same id (",(0,i.jsx)(t.em,{children:"Guid"}),"), NUnit will report them as a single test case, and will merge all associated results in a single test result.\nI.e. multiple NUnit tests will be reported as one test with one result."]}),"\n",(0,i.jsx)(t.h3,{id:"ignored-tests-1",children:"Ignored tests"}),"\n",(0,i.jsxs)(t.p,{children:["Ignored test cases are discovered as normal, but the tests are not run, events are not raised for data collectors and they are reported as ",(0,i.jsx)(t.code,{children:"Skipped"}),"."]}),"\n",(0,i.jsx)(t.h3,{id:"testcases-with-variable-data",children:"TestCases with variable data"}),"\n",(0,i.jsxs)(t.p,{children:["NUnit's TestCases is a pattern where some test is executed multiple times with varying input data (like xUnit's Theory).\nThese test methods accept one or more parameters and bear one of these attributes (on top of the ",(0,i.jsx)(t.code,{children:"[Test]"})," attribute):\n",(0,i.jsx)(t.code,{children:"TestCase"})," or ",(0,i.jsx)(t.code,{children:"TestCaseSource"}),"."]}),"\n",(0,i.jsxs)(t.p,{children:["There are some differences\nbetween ",(0,i.jsx)(t.code,{children:"TestCase"})," (data is fixed at build time) and ",(0,i.jsx)(t.code,{children:"TestCaseSource"})," (data is discovered at run time)."]}),"\n",(0,i.jsx)(t.h3,{id:"testcase",children:"TestCase"}),"\n",(0,i.jsxs)(t.p,{children:["Basically, each ",(0,i.jsx)(t.code,{children:"TestCase"})," is considered to be a specific test, and NUnit generates a name for each of those using the\n",(0,i.jsx)(t.code,{children:"ToString()"})," method for each parameter (e.g: ",(0,i.jsx)(t.code,{children:"Test(1,2)"})," for parameter 1 and 2).\nThen each test is run separately and a test result is provided for each test.\n",(0,i.jsx)(t.strong,{children:"Caveat"}),": if the ",(0,i.jsx)(t.code,{children:"ToString()"})," method is not overridden, multiple test cases will have the same name\n(e.g.: ",(0,i.jsx)(t.code,{children:"Test(MyClass"}

1),"). Then, there will be no way to distinguish between tests as well as properly associate test\nresults."]}),"\n",(0,i.jsx)(t.h3,{id:"testcasesource",children:"TestCaseSource"}),"\n",(0,i.jsxs)(t.p,{children:["Each test case is reported during the discovery phase, and NUnit generates a name for each of those using the\n",(0,i.jsx)(t.code,{children:"ToString()"})," method for each parameter (e.g: ",(0,i.jsx)(t.code,{children:"Test(1,2)"})," for parameter 1 and 2).\nThen each test is run separately and a test result is provided for each test.\n",(0,i.jsx)(t.strong,{children:"Caveat"}),": if the ",(0,i.jsx)(t.code,{children:"ToString()"})," method is not overridden, multiple test cases will have the same name\n(e.g.: ",(0,i.jsx)(t.code,{children:"Test(MyClass"}),"). Then, there will be no way to distinguish between tests as well as properly associate test\nresults.\nWhen this happens, NUnit will run the test against all data in the set and return a single result (failed if at least one test failed).\nThe drawback is that you can't identify which data set failed the test."]}),"\n",(0,i.jsx)(t.h3,{id:"run-sequence-1",children:"Run Sequence"}),"\n",(0,i.jsx)(t.p,{children:"Here is a summarized timeline of tests execution:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{children:"NUnit runner calls the TestCaseSource and gets every test case.\nTestSession start event\n ...\n   NUnit discovers test\n     NUnit enumerates data source (for tests)\n...\n  \nTestCase start event\n   NUnit run test with first set of data\n   NUnit run test with second set of data\n   ...\n   NUnit reports test result\nTestCase end event\n   ...\n   NUnit reports last test result\nTestCase end event\nTestCase start event\n   ...\n"})}),"\n",(0,i.jsx)(t.p,{children:"The problems for Stryker are:"}),"\n",(0,i.jsxs)(t.ol,{children:["\n",(0,i.jsxs)(t.li,{children:["If test cases have the same name, we cannot distinguish between test cases, so it must be handled as a single test. This ",(0,i.jsx)(t.em,{children:"single test"})," reports multiple test cases so Stryker needs to wait either for one failure or for each testcase\nto succeed to establish the correct status. The good news is that discovery should provide the number of results to wait for."]}),"\n",(0,i.jsxs)(t.li,{children:["Every mutation encountered by the ",(0,i.jsx)(t.code,{children:"TestCaseSource"})," method will be reported as covered by the first test case,\nwhereas it is likely these mutations would need to be adequately associated with their relative test case."]}),"\n",(0,i.jsxs)(t.li,{children:["TestCaseSource is called ",(0,i.jsx)(t.strong,{children:"before"})," Testcase start event which means Stryker may set the active mutation too late.\nThe only way to handle this is to force any concerned mutants to run in dedicated test sessions."]}),"\n"]})]})}function h(e={}){let{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(l,{...e})}):l(e)}},67008:function(e,t,s){s.d(t,{R:function(){return a},x:function(){return o}});var n=s(71763);let i={},r=n.createContext(i);function a(e){let t=n.useContext(r);return n.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:a(e.components),n.createElement(r.Provider,{value:t},e.children)}}}]);