https://docs-production-katalon.netlify.app/assets/js/34ddfa5c.701a6a88.js

1"use strict";(self.webpackChunkKatalon_Docs=self.webpackChunkKatalon_Docs||[]).push([[6301],{28453:(e,s,n)=>{n.d(s,{R:()=>r,x:()=>l});var t=n(96540);const i={},a=t.createContext(i);function r(e){const s=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(s):{...s,...e}}),[s,e])}function l(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:r(e.components),t.createElement(a.Provider,{value:s},e.children)}},56253:(e,s,n)=>{n.r(s),n.d(s,{assets:()=>o,contentTitle:()=>l,default:()=>d,frontMatter:()=>r,metadata:()=>t,toc:()=>c});const t=JSON.parse('{"id":"analyze/probabilistic-flakiness-score","title":"Probabilistic Flakiness Score (PFS)","description":"This document explains how the Probabilistic Flakiness Score (PFS) is calculated to determine flaky tests in Katalon True Platform.","source":"@site/docs/katalon-platform/analyze/probabilistic-flakiness-score.md","sourceDirName":"analyze","slug":"/analyze/probabilistic-flakiness-score","permalink":"/katalon-platform/analyze/probabilistic-flakiness-score","draft":false,"unlisted":false,"tags":[],"version":"current","lastUpdatedAt":1775545163000,"frontMatter":{"title":"Probabilistic Flakiness Score (PFS)"},"sidebar":"katalon-platform","previous":{"title":"Test quality metrics","permalink":"/katalon-platform/analytics/references/test-quality-metrics"},"next":{"title":"Visual Testing overview","permalink":"/katalon-platform/analyze/visual-testing/visual-testing-overview"}}');var i=n(74848),a=n(28453);const r={title:"Probabilistic Flakiness Score (PFS)"},l=void 0,o={},c=[{value:"Overview",id:"overview",level:2},{value:"Calculations",id:"calculations",level:2},{value:"1. Flip rate: behavior changes across runs",id:"1-flip-rate-behavior-changes-across-runs",level:3},{value:"2. Retry rate: the need for retries",id:"2-retry-rate-the-need-for-retries",level:3},{value:"3. Probability of flakiness (two independent views)",id:"3-probability-of-flakiness-two-independent-views",level:3},{value:"a. Time-based instability",id:"a-time-based-instability",level:4},{value:"b. Balance of passes vs failures (adjusted for uncertainty)",id:"b-balance-of-passes-vs-failures-adjusted-for-uncertainty",level:4},{value:"4. Uncertainty interval",id:"4-uncertainty-interval",level:3},{value:"Combining everything into the final score",id:"combining-everything-into-the-final-score",level:3}];function h(e){const s={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(s.p,{children:["This document explains how the ",(0,i.jsx)(s.strong,{children:"Probabilistic Flakiness Score (PFS)"})," is calculated to determine flaky tests in ",(0,i.jsx)(s.strong,{children:"Katalon True Platform"}),"."]}),"\n",(0,i.jsx)(s.h2,{id:"overview",children:"Overview"}),"\n",(0,i.jsxs)(s.p,{children:["Detecting flaky tests is one of ",(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/configure-smart-tags",children:"Smart Tags"}),"' four core features, calculated based on the ",(0,i.jsx)(s.strong,{children:"PFS (Probabilistic Flakiness Score)"}),"."]}),"\n",(0,i.jsx)(s.p,{children:"The PFS is a statistical measure designed to capture and signal instability in test behavior over time, in addition to failure rate. It combines multiple flakiness indicators and weighs them based on how strongly they correlate with unreliable tests."}),"\n",(0,i.jsx)(s.p,{children:"Behind the scenes, we take into consideration the signals below:"}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.strong,{children:"Flip rate"}),": how often the test case flips between passing and failing."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.strong,{children:"Retry rate"}),": how often the test case retried."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.strong,{children:"Probability of flakiness:"})," measured by ",(0,i.jsx)(s.strong,{children:"time-based instability"})," and ",(0,i.jsx)(s.strong,{children:"pass/failure balance"}),"."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.strong,{children:"Uncertainty interval:"})," ",(0,i.jsx)(s.strong,{children:"Katalon True Platform' confidence level"})," for the score, based on how much data is available."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["All of these signals are combined to produce a single ",(0,i.jsx)(s.strong,{children:"PFS"}),", that truly reflects behavior over time and becomes more reliable as more runs are observed. The score should falls between ",(0,i.jsx)(s.code,{children:"0.0"})," and ",(0,i.jsx)(s.code,{children:"1.0"}),", with ",(0,i.jsx)(s.code,{children:"0.0"})," being most stable, and ",(0,i.jsx)(s.code,{children:"1.0"})," being highly flaky."]}),"\n",(0,i.jsx)(s.h2,{id:"calculations",children:"Calculations"}),"\n",(0,i.jsx)(s.p,{children:"Below are details on the calculations of the PFS:"}),"\n",(0,i.jsx)(s.h3,{id:"1-flip-rate-behavior-changes-across-runs",children:"1. Flip rate: behavior changes across runs"}),"\n",(0,i.jsxs)(s.p,{children:["This indicator shows ",(0,i.jsx)(s.strong,{children:"how often the test changes its outcome from one run to the next"}),"."]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"If a test passes consistently or fails consistently, it is considered stable."}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"If it frequently switches between pass and fail across consecutive runs, that instability increases the score."}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["This captures ",(0,i.jsx)(s.em,{children:"volatility over time"}),", not overall success rate."]}),"\n",(0,i.jsx)(s.h3,{id:"2-retry-rate-the-need-for-retries",children:"2. Retry rate: the need for retries"}),"\n",(0,i.jsxs)(s.p,{children:["This indicator shows ",(0,i.jsx)(s.strong,{children:"how often the test has to be rerun"})," to pass."]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"A test that frequently needs retries is treated as less trustworthy."}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"Even if retries eventually pass, the need to rerun suggests unreliable behavior."}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["This measures ",(0,i.jsx)(s.em,{children:"operational instability"})," rather than correctness."]}),"\n",(0,i.jsx)(s.h3,{id:"3-probability-of-flakiness-two-independent-views",children:"3. Probability of flakiness (two independent views)"}),"\n",(0,i.jsxs)(s.p,{children:["Flakiness probability is estimated using ",(0,i.jsx)(s.strong,{children:"two approaches"}),". Whichever method returns a worse result is used as the final probability for flakiness."]}),"\n",(0,i.jsx)(s.h4,{id:"a-time-based-instability",children:"a. Time-based instability"}),"\n",(0,i.jsxs)(s.p,{children:["This looks at ",(0,i.jsx)(s.strong,{children:"how unpredictable the pass/fail pattern is across runs"}),"."]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"A test whose outcomes vary randomly over time is considered more flaky."}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"A test with a clear pattern (mostly pass or mostly fail) is considered less flaky."}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["This captures ",(0,i.jsx)(s.em,{children:"randomness"}),", not averages."]}),"\n",(0,i.jsx)(s.h4,{id:"b-balance-of-passes-vs-failures-adjusted-for-uncertainty",children:"b. Balance of passes vs failures (adjusted for uncertainty)"}),"\n",(0,i.jsxs)(s.p,{children:["This looks at ",(0,i.jsx)(s.strong,{children:"how close the test is to a 50/50 pass\u2013fail split"}),":"]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"Tests that pass about half the time are treated as more flaky."}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"Tests that almost always pass or almost always fail are treated as less flaky."}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["When there are only a few runs, extra uncertainty (",(0,i.jsx)(s.strong,{children:"uncertainty interval"}),") is added as a weight to reflect less certainty on the true. It prevents early conclusions based on sparse data."]}),"\n",(0,i.jsxs)(s.p,{children:["The flakiness estimate is the ",(0,i.jsx)(s.strong,{children:"stronger of these two signals"}),", so flakiness is never underestimated."]}),"\n",(0,i.jsx)(s.h3,{id:"4-uncertainty-interval",children:"4. Uncertainty interval"}),"\n",(0,i.jsxs)(s.p,{children:["The uncertainty interval represents ",(0,i.jsx)(s.strong,{children:"Katalon True Platform' confidence level,"})," based solely on how much data is available."]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"When the number of runs is small, observed pass and fail counts may not accurately reflect the test's true behavior - uncertainty is high."}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsx)(s.p,{children:"As the total number of runs increases, the uncertainty interval gradually shrinks. With more observations, the system becomes more confident in the test's true stability, rather than random chance."}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["The uncertainty interval quantifies ",(0,i.jsx)(s.strong,{children:"how much trust the system places in the observed results"}),", based on the amount of information available rather than the outcomes themselves."]}),"\n",(0,i.jsx)(s.h3,{id:"combining-everything-into-the-final-score",children:"Combining everything into the final score"}),"\n",(0,i.jsxs)(s.p,{children:["Finally, all indicators are ",(0,i.jsx)(s.strong,{children:"combined into a single PFS"})," that ",(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/configure-smart-tags",children:(0,i.jsx)(s.strong,{children:"Smart Tags"})})," use to flag a test case as \u201cstable\u201d, \u201cpotentially flaky\u201d, or \u201chighly flaky\u201d:"]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.code,{children:"0.0"})," - ",(0,i.jsx)(s.code,{children:"0.2"}),": Consistent tests. These tests are reliable, with predictable behaviour."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.code,{children:"0.2"})," - ",(0,i.jsx)(s.code,{children:"0.8"}),": Potentially flaky tests. These tests are inconsistent and could potentially cause development friction."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.code,{children:"0.8"})," - ",(0,i.jsx)(s.code,{children:"1.0"}),": Highly flaky tests. These tests are severely unreliable with frequent status changes, high retry rates, or persistent failures requiring immediate attention."]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(s.p,{children:["You can see the ",(0,i.jsx)(s.code,{children:"Flaky"})," tag and the ",(0,i.jsx)(s.strong,{children:"Flakiness"})," widget on:"]}),"\n",(0,i.jsxs)(s.ul,{children:["\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.strong,{children:"Executions"})," tab: flaky test runs on this page are flagged ",(0,i.jsx)(s.code,{children:"Flaky"}),"; each is linked to its ",(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/view-test-results#view-test-run-details",children:(0,i.jsx)(s.strong,{children:"Test Run Details page"})})," for further investigation."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/about-the-analytics-and-trends-dashboard",children:(0,i.jsx)(s.strong,{children:"Analytics & Trends dashboard"})}),": this dashboard\u2019s ",(0,i.jsx)(s.code,{children:"Flakiness"})," widget displays the number of flaky tests in the selected scope/filters."]}),"\n"]}),"\n",(0,i.jsxs)(s.li,{children:["\n",(0,i.jsxs)(s.p,{children:[(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/reports/test-case-status-analysis-report",children:(0,i.jsx)(s.strong,{children:"Test Case Health Analysis Report"})}),": this report has the ",(0,i.jsx)(s.code,{children:"Flakiness"})," widget and a data table with links to each flaky test's ",(0,i.jsx)(s.em,{children:"latest"})," ",(0,i.jsx)(s.a,{href:"/katalon-platform/analyze/view-test-results#view-test-result-details",children:(0,i.jsx)(s.strong,{children:"Test Results Details Page"})}),", for investigation purposes."]}),"\n"]}),"\n"]})]})}function d(e={}){const{wrapper:s}={...(0,a.R)(),...e.components};return s?(0,i.jsx)(s,{...e,children:(0,i.jsx)(h,{...e})}):h(e)}}}]);