1"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5816],{28453(e,n,s){s.d(n,{R:()=>l,x:()=>o});var a=s(96540);const t={},i=a.createContext(t);function l(e){const n=a.useContext(i);return a.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:l(e.components),a.createElement(i.Provider,{value:n},e.children)}},86517(e,n,s){s.r(n),s.d(n,{assets:()=>r,contentTitle:()=>o,default:()=>h,frontMatter:()=>l,metadata:()=>a,toc:()=>c});const a=JSON.parse('{"id":"analyses","title":"Utilizing and Creating Analyses","description":"This document discusses non-API components of Ax, which may change between major","source":"@site/versioned_docs/version-1.2.4/analyses.md","sourceDirName":".","slug":"/analyses","permalink":"/docs/1.2.4/analyses","draft":false,"unlisted":false,"tags":[],"version":"1.2.4","lastUpdatedBy":"github-actions[bot]","lastUpdatedAt":1772675419000,"frontMatter":{"id":"analyses","title":"Utilizing and Creating Analyses"}}');var t=s(74848),i=s(28453);const l={id:"analyses",title:"Utilizing and Creating Analyses"},o="Utilizing and Creating Ax Analyses",r={},c=[{value:"Using Analyses",id:"using-analyses",level:2},{value:"Creating a new Analysis",id:"creating-a-new-analysis",level:2},{value:"Adding options to an Analysis",id:"adding-options-to-an-analysis",level:2},{value:"Miscellaneous tips",id:"miscellaneous-tips",level:2}];function d(e){const n={admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",header:"header",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.admonition,{type:"info",children:(0,t.jsx)(n.p,{children:"This document discusses non-API components of Ax, which may change between major\nlibrary versions. Contributor guides are most useful for developers intending to\npublish PRs to Ax, not those using Ax directly or building tools on top of Ax."})}),"\n",(0,t.jsx)(n.header,{children:(0,t.jsx)(n.h1,{id:"utilizing-and-creating-ax-analyses",children:"Utilizing and Creating Ax Analyses"})}),"\n",(0,t.jsxs)(n.p,{children:["Ax\u2019s Analysis module provides a framework for producing plots, tables, messages,\nand more to help users understand their experiments. This is facilitated via the\n",(0,t.jsx)(n.code,{children:"Analysis"})," protocol and its various subclasses."]}),"\n",(0,t.jsxs)(n.p,{children:["Analysis classes implement a method ",(0,t.jsx)(n.code,{children:"compute"})," which consumes an ",(0,t.jsx)(n.code,{children:"Experiment"}),",\n",(0,t.jsx)(n.code,{children:"GenerationStrategy"}),", and/or ",(0,t.jsx)(n.code,{children:"Adapter"})," and outputs a collection of\n",(0,t.jsx)(n.code,{children:"AnalysisCards"}),". These cards contain a dataframe with relevant data, a \u201cblob\u201d\nwhich contains data to be rendered (ex. a plot), and miscellaneous metadata like\na title, subtitle, and priority level used for sorting. ",(0,t.jsx)(n.code,{children:"compute"})," returns a\ncollection of cards so that Analyses can be composed together. For example: the\n",(0,t.jsx)(n.code,{children:"TopSurfacesPlot"})," computes a ",(0,t.jsx)(n.code,{children:"SensitivityAnalysisPlot"})," to understand which\nparameters in the search space are most relevent, then produces ",(0,t.jsx)(n.code,{children:"SlicePlot"}),"s and\n",(0,t.jsx)(n.code,{children:"ContourPlot"}),"s for the most important surfaces."]}),"\n",(0,t.jsxs)(n.p,{children:["Ax currently provides implementations for 3 base classes: (1)",(0,t.jsx)(n.code,{children:"Analysis"})," -- for\ncreating tables, (2) ",(0,t.jsx)(n.code,{children:"PlotlyAnalysis"})," -- for producing plots using the Plotly\nlibrary, and (3) ",(0,t.jsx)(n.code,{children:"MarkdownAnalysis"})," -- for producing messages. Importantly Ax is\nable to save these cards to the database using ",(0,t.jsx)(n.code,{children:"save_analysis_cards"}),", allowing\nfor analyses to be pre-computed and displayed at a later time. This is done\nautomatically when ",(0,t.jsx)(n.code,{children:"Client.compute_analyses"})," is called."]}),"\n",(0,t.jsx)(n.h2,{id:"using-analyses",children:"Using Analyses"}),"\n",(0,t.jsxs)(n.p,{children:["The simplest way to use an ",(0,t.jsx)(n.code,{children:"Analysis"})," is to call ",(0,t.jsx)(n.code,{children:"Client.compute_analyses"}),". This\nwill heuristically select the most relevant analyses to compute, save the cards\nto the database, return them, and display them in your IPython environment if\npossible. Users can also specify which anal
1yses to compute and pass them in\nmanually, for example:\n",(0,t.jsx)(n.code,{children:"client.compute_analyses(analyses=[TopSurfacesPlot(), Summary(), ...])"}),"."]}),"\n",(0,t.jsxs)(n.p,{children:["When developing a new ",(0,t.jsx)(n.code,{children:"Analysis"}),' it can be useful to compute an analysis "a-la\ncarte". To do this, manually instantiate the ',(0,t.jsx)(n.code,{children:"Analysis"})," and call its ",(0,t.jsx)(n.code,{children:"compute"}),"\nmethod. This will return a collection of ",(0,t.jsx)(n.code,{children:"AnalysisCards"})," which can be displayed."]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:"analysis = CrossValidationPlot()\n\ncards = analysis.compute(\n experiment=experiment,\n generation_strategy=generation_strategy,\n adapter=adapter,\n)\n"})}),"\n",(0,t.jsx)(n.h2,{id:"creating-a-new-analysis",children:"Creating a new Analysis"}),"\n",(0,t.jsxs)(n.p,{children:["Let's implement a simple Analysis that returns a table counting the number of\ntrials in each ",(0,t.jsx)(n.code,{children:"TrialStatus"})," . We'll make a new class that implements the\n",(0,t.jsx)(n.code,{children:"Analysis"})," protocol (i.e. it defines a ",(0,t.jsx)(n.code,{children:"compute"})," method)."]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:'class TrialStatusTable(Analysis):\n def compute(\n self,\n experiment: Experiment | None = None,\n generation_strategy: GenerationStrategy | None = None,\n adapter: Adapter | None = None,\n ) -> Sequence[AnalysisCard]:\n trials_by_status = experiment.trials_by_status\n\n records = [\n {"status": status.name, "count": len(trials)}\n for status, trials in trials_by_status.items()\n ]\n\n return [\n self._create_analysis_card(\n title="Trials by Status",\n subtitle="How many trials are in each status?",\n level=AnalysisCardLevel.LOW,\n category=AnalysisCardCategory.INSIGHT,\n df=pd.DataFrame.from_records(records),\n )\n ]\n\ncards = client.compute_analyses(analyses=[TrialStatusTable()])\n'})}),"\n",(0,t.jsx)(n.h2,{id:"adding-options-to-an-analysis",children:"Adding options to an Analysis"}),"\n",(0,t.jsxs)(n.p,{children:["Imagine we wanted to add an option to change how this analysis is computed, say\nwe wish to toggle whether the analysis computes the ",(0,t.jsx)(n.em,{children:"number"})," of trials in a\ngiven state or the ",(0,t.jsx)(n.em,{children:"percentage"})," of trials in a given state. We cannot change the\ninput arguments to ",(0,t.jsx)(n.code,{children:"compute"}),", so this must be added elsewhere."]}),"\n",(0,t.jsxs)(n.p,{children:["The analysis' initializer is a natural place to put additional settings. We'll\ncreate a ",(0,t.jsx)(n.code,{children:"TrialStatusTable.__init__"})," method which takes in the option as a\nboolean, then modify ",(0,t.jsx)(n.code,{children:"compute"})," to consume this option as well. Following this\npatterns allows users to specify all relevant settings before calling\n",(0,t.jsx)(n.code,{children:"Client.compute_analyses"})," while still allowing the underlying ",(0,t.jsx)(n.code,{children:"compute"})," call to\nremain unchanged. Standarization of the ",(0,t.jsx)(n.code,{children:"compute"})," call simplifies logic\nelsewhere in the stack."]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-python",children:'class TrialStatusTable(Analysis):\n def __init__(self, as_fraction: bool) -> None:\n super().__init__()\n\n self.as_fraction = as_fraction\n\n def compute(\n self,\n experiment: Experiment | None = None,\n generation_strategy: GenerationStrategy | None = None,\n adapter: Adapter | None = None,\n ) -> Sequence[AnalysisCard]:\n trials_by_status = experiment.trials_by_status\n denominator = len(experiment.trials) if self.as_fraction else 1\n\n records = [\n {"status": status.name, "count": len(trials) / denominator}\n for status, trials in trials_by_status.items()\n ]\n\n return [\n # Use _create_analysis_card rather than AnalysisCard to automatically populate relevant metadata\n self._create_analysis_card(\n title="Trials by Status",\n subtitle="How many trials are in each status?",\n level=AnalysisCardLevel.LOW,\n category=A
1nalysisCardCategory.INSIGHT,\n df=pd.DataFrame.from_records(records),\n )\n ]\n\n\ncards = client.compute_analyses(analyses=[TrialStatusTable(as_fraction=True)])\n'})}),"\n",(0,t.jsx)(n.h2,{id:"miscellaneous-tips",children:"Miscellaneous tips"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:["Many analyses rely on the same infrastructure and utility functions -- check\nto see if what you need has already been implemented somewhere.","\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:["Many analyses require an ",(0,t.jsx)(n.code,{children:"Adapter"})," but can use either the ",(0,t.jsx)(n.code,{children:"Adapter"})," provided\nor the current ",(0,t.jsx)(n.code,{children:"Adapter"})," on the ",(0,t.jsx)(n.code,{children:"GenerationStrategy"})," --\n",(0,t.jsx)(n.code,{children:"extract_relevant_adapter"})," handles this in a consistent way"]}),"\n",(0,t.jsxs)(n.li,{children:["Analyses which use an ",(0,t.jsx)(n.code,{children:"Arm"})," as the fundamental unit of analysis will find\nthe ",(0,t.jsx)(n.code,{children:"prepare_arm_data"})," utility useful; using it will also lend the\n",(0,t.jsx)(n.code,{children:"Analysis"})," useful features like relativization for free"]}),"\n"]}),"\n"]}),"\n",(0,t.jsxs)(n.li,{children:["When writing a new ",(0,t.jsx)(n.code,{children:"PlotlyAnalysis"})," check out ",(0,t.jsx)(n.code,{children:"ax.analysis.plotly.utils"})," for\nguidance on using color schemes and unified tool tips"]}),"\n",(0,t.jsxs)(n.li,{children:["Try to follow consistent design patterns; many analyses take an optional list\nof ",(0,t.jsx)(n.code,{children:"metric_names"})," on initialization, and interpret ",(0,t.jsx)(n.code,{children:"None"})," to mean the user\nwants to compute a card for each metric present. Following these conventions\nmakes things easier for downstream consumers."]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(d,{...e})}):d(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.