https://ax.dev/assets/js/74816b42.788803e7.js

1"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3689],{11135(e,n,i){i.r(n),i.d(n,{assets:()=>m,contentTitle:()=>p,default:()=>f,frontMatter:()=>x,metadata:()=>t,toc:()=>j});const t=JSON.parse('{"id":"experiment","title":"Experiment + Trials","description":"This document discusses non-API components of Ax, which may change between major","source":"@site/versioned_docs/version-1.3.1/experiment.mdx","sourceDirName":".","slug":"/experiment","permalink":"/docs/experiment","draft":false,"unlisted":false,"tags":[],"version":"1.3.1","lastUpdatedBy":"github-actions[bot]","lastUpdatedAt":1781025267000,"frontMatter":{"id":"experiment","title":"Experiment + Trials"},"sidebar":"docs","previous":{"title":"Introduction to Bayesian Optimization","permalink":"/docs/intro-to-bo"},"next":{"title":"Orchestration","permalink":"/docs/orchestration"}}');var a=i(74848),s=i(28453);const r=i.p+"assets/images/ask_tell_simple-3301c4e52b243509ebe901605c27189b.png",o=i.p+"assets/images/ask_tell_flowchart-9aa1655dde35b8c3ffec25b6c444f236.png",c=i.p+"assets/images/experiment_composition-7a550eeda7b1ca4c1a7c94003d9ff5d4.png",l=i.p+"assets/images/trial_composition-246e5f4f0cde8e1854253fdf4a5f804a.png",d=i.p+"assets/images/search_space_composition-5ce5d77e9e2b0288dc8edf58666f5c1f.png",h=i.p+"assets/images/optimization_config_composition-746d95b0e60ab14b570da454562ccc62.png",x={id:"experiment",title:"Experiment + Trials"},p="Experiment and its components: Trial, Arm, SearchSpace, OptimizationConfig",m={},j=[{value:"Overview",id:"overview",level:2},{value:"Experiment",id:"experiment",level:2},{value:"Trial and Batch Trial",id:"trial-and-batch-trial",level:2},{value:"Trial Lifecycle and Status",id:"trial-lifecycle-and-status",level:3},{value:"SearchSpace and Parameters",id:"searchspace-and-parameters",level:2},{value:"Parameter Constraints",id:"parameter-constraints",level:3},{value:"Can I have parameter constraints on my objective?",id:"can-i-have-parameter-constraints-on-my-objective",level:4},{value:"What about non-linear constraints?",id:"what-about-non-linear-constraints",level:4},{value:"What about equality constraints?",id:"what-about-equality-constraints",level:4},{value:"OptimizationConfig",id:"optimizationconfig",level:2},{value:"Can Ax still create trials that will violate constraints?",id:"can-ax-still-create-trials-that-will-violate-constraints",level:4},{value:"Further Reading",id:"further-reading",level:2}];function u(e){const n={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",h4:"h4",header:"header",li:"li",ol:"ol",p:"p",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.admonition,{type:"info",children:(0,a.jsx)(n.p,{children:"This document discusses non-API components of Ax, which may change between major\nlibrary versions. Contributor guides are most useful for developers intending to\npublish PRs to Ax, not those using Ax directly or building tools on top of Ax."})}),"\n",(0,a.jsx)(n.header,{children:(0,a.jsxs)(n.h1,{id:"experiment-and-its-components-trial-arm-searchspace-optimizationconfig",children:[(0,a.jsx)(n.code,{children:"Experiment"})," and its components: ",(0,a.jsx)(n.code,{children:"Trial"}),", ",(0,a.jsx)(n.code,{children:"Arm"}),", ",(0,a.jsx)(n.code,{children:"SearchSpace"}),", ",(0,a.jsx)(n.code,{children:"OptimizationConfig"})]})}),"\n",(0,a.jsxs)(n.p,{children:["As we discuss in ",(0,a.jsx)(n.a,{href:"/docs/intro-to-ae",children:"Intro to Adaptive Experimentation"}),", every\noptimization in Ax is an iterative process where we:"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["Generate candidate datapoints to evaluate (represented by ",(0,a.jsx)(n.code,{children:"Trial"}),"s)"]}),"\n",(0,a.jsx)(n.li,{children:"Learn from datapoints we have observed"}),"\n",(0,a.jsxs)(n.li,{children:["In order to find an optimal point (",(0,a.jsx)(n.code,{children:"Arm"})," in Ax) by ",(0,a.jsx)(n.em,{children:"balancing"}),":","\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.em,{children:"exploration"})," (learning more about the behavior of outcomes (",(0,a.jsx)(n.code,{children:"Metric"}),"s) in\nresponse to a change in ",(0,a.jsx)(n.code,{children:"Parameter"})," values)"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.em,{children:"and exploitation"})," (leveraging the knowledge we gained, to identify likely\noptimal points/",(0,a.jsx)(n.code,{children:"Arm"}),"s)."]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:r,alt:"Using Ax for 'ask-tell' optimization",width:"60%"})}),"\n",(0,a.jsx)(n.h2,{id:"overview",children:"Overview"}),"\n",(0,a.jsx)(n.p,{children:"In the Ax data model, this process is represented through three high-order\ncomponents:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"Experiment"}),": keeps track of the whole optimization process and its state,"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"GenerationStrategy"}),": contains all the information about what methodology Ax\nwill use to produce the next ",(0,a.jsx)(n.code,{children:"Arm"}),"s to try in the course of the ",(0,a.jsx)(n.code,{children:"Experiment"}),","]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"Orchestrator"})," (optional): conducts a full experiment with automatic trial\ndeployment and data fetching given an ",(0,a.jsx)(n.code,{children:"Experiment"})," and a ",(0,a.jsx)(n.code,{children:"GenerationStrategy"}),"\nobjects (and a set of optional configurations that make the orchestration\nflexible and configurable)."]}),"\n"]}),"\n",(0,a.jsxs)(n.p,{children:["Users interact with ",(0,a.jsx)(n.code,{children:"Experiment"})," and ",(0,a.jsx)(n.code,{children:"GenerationStrategy"})," objects through\nmethods like ",(0,a.jsx)(n.code,{children:"Client.get_next_trials"})," and ",(0,a.jsx)(n.code,{children:"Client.complete_trial"}),"), and\noptionally an ",(0,a.jsx)(n.code,{children:"Orchestrator"})," with ",(0,a.jsx)(n.code,{children:"Client.run_n_trials"}),". The iterative process\nof using the ",(0,a.jsx)(n.code,{children:"Client"})," looks like this in more detail:"]}

1),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:o,alt:"Using Ax for 'ask-tell' optimization",width:"80%"})}),"\n",(0,a.jsxs)(n.p,{children:["We recommend avoiding interacting with the ",(0,a.jsx)(n.code,{children:"Experiment"})," object directly unless\ndeveloping Ax internals (opting to interact with it through ",(0,a.jsx)(n.code,{children:"Client"})," instead),\nbut understanding its structure can help conceptualize all the data tracked and\nleveraged in Ax."]}),"\n",(0,a.jsx)(n.h2,{id:"experiment",children:"Experiment"}),"\n",(0,a.jsxs)(n.p,{children:["An Ax ",(0,a.jsx)(n.code,{children:"Experiment"})," is composed of:"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsxs)(n.strong,{children:["A collection of indexed ",(0,a.jsx)(n.code,{children:"Trial"}),"s"]})," (or ",(0,a.jsx)(n.code,{children:"BatchTrial"}),"s, ",(0,a.jsx)(n.a,{href:"#trial-and-batch-trial",children:"more on these\nbelow"}),"), each of which contain one or more ",(0,a.jsx)(n.code,{children:"Arm"}),'s (representing a point that\nwas "tried" in the course of the ',(0,a.jsx)(n.code,{children:"Experiment"}),").","\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["Each trial records metadata about one evaluation of its ",(0,a.jsx)(n.code,{children:"Arm"}),"s, in the form\nof ",(0,a.jsx)(n.code,{children:"Data"}),". In a noisy setting, multiple ",(0,a.jsx)(n.code,{children:"Trial"}),"s with the same ",(0,a.jsx)(n.code,{children:"Arm"}),"s\nmight produce different data; Ax optimization algorithms excel in such\nsettings."]}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsxs)(n.strong,{children:["Information about the ",(0,a.jsx)(n.code,{children:"Experiment"})," design"]}),", e.g. ",(0,a.jsx)(n.code,{children:"SearchSpace"})," Ax will be\nexploring and ",(0,a.jsx)(n.code,{children:"OptimizationConfig"})," that Ax will be targeting by optimizing\nobjectives and avoiding violation of constraints."]}),"\n"]}),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:c,alt:"An Experiment and the classes that comprise it",width:"60%"})}),"\n",(0,a.jsxs)(n.p,{children:["We use the ",(0,a.jsx)(n.code,{children:"Experiment"})," to keep track of the whole optimization process. It\ndescribes:"]}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:['"Where Ax should look" (via ',(0,a.jsx)(n.code,{children:"SearchSpace"}),", ",(0,a.jsx)(n.code,{children:"Parameter"}),"s and\n",(0,a.jsx)(n.code,{children:"ParameterConstraint"}),"s),"]}),"\n",(0,a.jsxs)(n.li,{children:['"What Ax should optimize for" (via ',(0,a.jsx)(n.code,{children:"OptimizationConfig"}),", composed of one or\nmultiple ",(0,a.jsx)(n.code,{children:"Objective"}),"s and ",(0,a.jsx)(n.code,{children:"OutcomeConstraint"}),"s),"]}),"\n",(0,a.jsxs)(n.li,{children:['"What have we tried so far in this experiment" (via ',(0,a.jsx)(n.code,{children:"Trial"}),"s and ",(0,a.jsx)(n.code,{children:"Data"}),"\nassociated with each of them),"]}),"\n",(0,a.jsxs)(n.li,{children:['Optionally "How do we run each trial and get its data" (via ',(0,a.jsx)(n.code,{children:"Runner"}),"s and\n",(0,a.jsx)(n.code,{children:"Metric"}),"s, typically applicable only if using Ax orchestration)."]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"trial-and-batch-trial",children:"Trial and Batch Trial"}),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:l,alt:"A Trial and the classes that comprise it",width:"60%"})}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsxs)(n.strong,{children:["An ",(0,a.jsx)(n.code,{children:"Experiment"})," is composed of a sequence of ",(0,a.jsx)(n.code,{children:"Trial"}),"s, each of which has\nparameterization(s) (or ",(0,a.jsx)(n.code,{children:"Arm"}),"-s) to be evaluated and a unique identifier: an\nindex."]})," A ",(0,a.jsx)(n.code,{children:"Trial"})," is added to the experiment when a new set of arms is proposed\nby the optimization algorithm (or manually attached by a user). The trial is\nthen evaluated to compute the values of each important outcome (or ",(0,a.jsx)(n.code,{children:"Metric"}),") for\neach arm, which are fed into the algorithms to create a new trial."]}),"\n",(0,a.jsxs)(n.p,{children:["A regular ",(0,a.jsx)(n.code,{children:"Trial"})," contains a single arm and relevant metadata. A ",(0,a.jsx)(n.code,{children:"BatchTrial"}),"\ncontains multiple arms, relevant metadata, and a set of arm weights, which are a\nmeasure of how much of the total resources allocated to evaluating a batch\nshould go towards evaluating the specific arm. ",(0,a.jsxs)(n.strong,{children:["The vast majority of Ax use\ncases will only need ",(0,a.jsx)(n.code,{children:"Trial"}

1)," and not ",(0,a.jsx)(n.code,{children:"BatchTrial"}),"."]})]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"A batch trial is not just a trial with many arms!"})," It is a trial for which\nit is important that the arms are evaluated jointly and ",(0,a.jsx)(n.em,{children:"together"}),". For\ninstance, a batch trial would be appropriate in an A/B test where the evaluation\nresults are subject to nonstationarity and require multiple arms to be deployed\n(and gathered data for) at the same time. ",(0,a.jsx)(n.strong,{children:"For cases where multiple arms are\nevaluated independently (even if concurrently), use multiple trials with a\nsingle arm each, which will allow Ax to keep track of them appropriately and\nselect an optimal optimization algorithm for this setting."})]}),"\n",(0,a.jsx)(n.h3,{id:"trial-lifecycle-and-status",children:"Trial Lifecycle and Status"}),"\n",(0,a.jsx)(n.p,{children:"A trial goes through multiple phases during the experimentation cycle:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"CANDIDATE"})," -- Trial has just been created and can still be modified before\ndeployment."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"STAGED"})," -- Relevant for external systems, where the trial configuration has\nbeen deployed but not begun the evaluation stage."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"RUNNING"})," -- Trial is in the process of being evaluated. Trials generated via\n",(0,a.jsx)(n.code,{children:"Client.get_next_trials"})," are in this status once the call to that method\nreturns."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"COMPLETED"})," -- Trial completed evaluation successfully."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"FAILED"})," -- Trial incurred a failure while being evaluated."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"ABANDONED"})," -- User manually stopped the trial for some specified reason."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.code,{children:"EARLY_STOPPED"})," -- Trial stopped before completion, likely based on\nintermediate data, and with use of an Ax ",(0,a.jsx)(n.code,{children:"EarlyStoppingStrategy"}),"."]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"searchspace-and-parameters",children:"SearchSpace and Parameters"}),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:d,alt:"A SearchSpace and the classes that comprise it",width:"60%"})}),"\n",(0,a.jsxs)(n.p,{children:["A search space is composed of a set of parameters to be tuned in the experiment,\nand optionally a set of parameter constraints that define restrictions across\nthese parameters (e.g. \u201cp_a <= p_b\u201d). Each parameter has a name, a type (",(0,a.jsx)(n.code,{children:"int"}),",\n",(0,a.jsx)(n.code,{children:"float"}),", ",(0,a.jsx)(n.code,{children:"bool"}),", or ",(0,a.jsx)(n.code,{children:"string"}),"), and a domain, which is a representation of the\npossible values the parameter can take. The search space is used by the\noptimization algorithms to know which arms are valid to suggest."]}),"\n",(0,a.jsx)(n.p,{children:"Ax supports three types of parameters:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Range parameters:"})," must be of type ",(0,a.jsx)(n.code,{children:"int"})," or ",(0,a.jsx)(n.code,{children:"float"}),", and the domain is\nrepresented by a lower and upper bound. If the parameter is specified as an\n",(0,a.jsx)(n.code,{children:"int"}),", newly generated points are rounded to the nearest integer by default."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Choice parameters:"})," domain is a set of values (values can be ",(0,a.jsx)(n.code,{children:"int"}),",\n",(0,a.jsx)(n.code,{children:"float"}),", ",(0,a.jsx)(n.code,{children:"bool"})," or ",(0,a.jsx)(n.code,{children:"string"}),")."]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"Fixed parameters:"})," domain is restricted to a single value (same types as\nChoice)."]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"parameter-constraints",children:"Parameter Constraints"}),"\n",(0,a.jsxs)(n.p,{children:["Ax supports linear parameter constraints which can be used on numerical (i.e.\n",(0,a.jsx)(n.code,{children:"int"})," or ",(0,a.jsx)(n.code,{children:"float"}),") parameters. These can take a number of forms, including order\nconstraints (ex. ",(0,a.jsx)(n.code,{children:"x1 <= x2"}),"), sum constraints (ex. ",(0,a.jsx)(n.code,{children:"x1 + x2 <= 1"}),"), or full\nweighted sums (ex. ",(0,a.jsx)(n.code,{children:"0.5 * x1 + 0.3 * x2 + ... <= 1"}),")."]}),"\n",(0,a.jsx)(n.h4,{id:"can-i-have-parameter-constraints-on-my-objective",children:"Can I have parameter constraints on my objective?"}),"\n",(0,a.jsx)(n.p,{children:'A constraint can only apply to an objective if Ax is conducting a\nmulti-objective optimization, and we call this special case of constraints\n"objective thresholds". These provide a "reference point" to Ax multi-objective\noptimization, informing Ax that trials where value of the objective is "worse"\nthan the objective threshold, are not part of the Pareto frontier we should be\nexploring. For example, if we are looking to jointly optimize model accuracy and\nsize, we might indicate that even the highest possible accuracy where model size\nis past a certain "feasibility threshold" (say, the maximum size model that can\nfind onto the target device) is no longer of interest in a given Ax\noptimization.'}),"\n",(0,a.jsx)(n.h4,{id:"what-about-non-linear-constraints",children:"What about non-linear constraints?"}),"\n",(0,a.jsx)(n.p,{children:"Non-linear parameter constraints are not supported by Ax at this time, due to\nchallenges in transforming them to the model space."}),"\n",(0,a.jsx)(n.h4,{id:"what-about-equality-constraints",children:"What about equality constraints?"}),"\n",(0,a.jsxs)(n.p,{children:["Ax does not currently support equality constraints. Often, search spaces which\ndesire equality constraints can be reparameterized to use order 

1constraints\ninstead. For example if we have a parameters ",(0,a.jsx)(n.code,{children:"x1"}),", ",(0,a.jsx)(n.code,{children:"x2"}),", and ",(0,a.jsx)(n.code,{children:"x3"})," and want to\nconstraint ",(0,a.jsx)(n.code,{children:"x1 + x2 + x3 = 1"})," the search space can be reparameterized to just\ndefine ",(0,a.jsx)(n.code,{children:"x1"})," and ",(0,a.jsx)(n.code,{children:"x2"})," with the inequality constraint ",(0,a.jsx)(n.code,{children:"x1 + x2 <= 1"}),", and the\nvalue ",(0,a.jsx)(n.code,{children:"1 - (x1 + x2)"})," can be substituted where ",(0,a.jsx)(n.code,{children:"x3"})," would have been used."]}),"\n",(0,a.jsx)(n.h2,{id:"optimizationconfig",children:"OptimizationConfig"}),"\n",(0,a.jsx)("center",{children:(0,a.jsx)("img",{src:h,alt:"An OptimizationConfig and the classes that comprise it",width:"60%"})}),"\n",(0,a.jsx)(n.p,{children:"An optimization config defines the goals of Ax optimization, e.g. \u201cmaximize NE\nwhile minimizing model size and avoiding a regression in model calibration\u201d. It\nis composed of:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["One or more ",(0,a.jsx)(n.strong,{children:"objectives to be minimized or maximized"}),","]}),"\n",(0,a.jsxs)(n.li,{children:["Optionally ",(0,a.jsx)(n.strong,{children:"a set of outcome constraints"})," that place restrictions on how other\nmetrics can be moved by the experiment:","\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.strong,{children:"A constraint can only apply to an objective if Ax is conducting a\nmulti-objective optimization"}),"; we call such \u201cconstraints on objectives\u201d\n",(0,a.jsx)(n.strong,{children:"objective thresholds"}),". These provide a \u201creference point\u201d to Ax\nmulti-objective optimization, informing Ax that trials where value of the\nobjective is \u201cworse\u201d than the objective threshold, are not part of the\nPareto frontier we should be exploring. E.g. if we are looking to\nco-optimize model accuracy and size, we might indicate that even the highest\npossible accuracy where model size is past a certain \u201cfeasibility threshold\u201d\nis no longer of interest in a given Ax optimization."]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h4,{id:"can-ax-still-create-trials-that-will-violate-constraints",children:"Can Ax still create trials that will violate constraints?"}),"\n",(0,a.jsx)(n.p,{children:"Yes, since Ax is aiming to predict constraint violations, but its predictions\nwon\u2019t always be correct. By definition, Ax is proposing next trials before\nreceiving their data, so the measurements of metric values found during the\nevaluation of a given trial, could differ from the \u201cexpectation\u201d of the Ax\noptimizers, especially earlier in the course of an experiment."}),"\n",(0,a.jsx)(n.h2,{id:"further-reading",children:"Further Reading"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"/docs/orchestration",children:"Internal Organization of Ax: Orchestration"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"/docs/generation_strategy",children:"Internal Organization of Ax: GenerationStrategy"})}),"\n"]})]})}function f(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(u,{...e})}):u(e)}},28453(e,n,i){i.d(n,{R:()=>r,x:()=>o});var t=i(96540);const a={},s=t.createContext(a);function r(e){const n=t.useContext(s);return t.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:r(e.components),t.createElement(s.Provider,{value:n},e.children)}}}]);