1"use strict";(self.webpackChunkpatternize_io=self.webpackChunkpatternize_io||[]).push([[4727],{15680:(a,e,s)=>{s.d(e,{xA:()=>g,yg:()=>o});var m=s(96540);function n(a,e,s){return e in a?Object.defineProperty(a,e,{value:s,enumerable:!0,configurable:!0,writable:!0}):a[e]=s,a}function t(a,e){var s=Object.keys(a);if(Object.getOwnPropertySymbols){var m=Object.getOwnPropertySymbols(a);e&&(m=m.filter((function(e){return Object.getOwnPropertyDescriptor(a,e).enumerable}))),s.push.apply(s,m)}return s}function p(a){for(var e=1;e<arguments.length;e++){var s=null!=arguments[e]?arguments[e]:{};e%2?t(Object(s),!0).forEach((function(e){n(a,e,s[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(a,Object.getOwnPropertyDescriptors(s)):t(Object(s)).forEach((function(e){Object.defineProperty(a,e,Object.getOwnPropertyDescriptor(s,e))}))}return a}function r(a,e){if(null==a)return{};var s,m,n=function(a,e){if(null==a)return{};var s,m,n={},t=Object.keys(a);for(m=0;m<t.length;m++)s=t[m],e.indexOf(s)>=0||(n[s]=a[s]);return n}(a,e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(a);for(m=0;m<t.length;m++)s=t[m],e.indexOf(s)>=0||Object.prototype.propertyIsEnumerable.call(a,s)&&(n[s]=a[s])}return n}var l=m.createContext({}),N=function(a){var e=m.useContext(l),s=e;return a&&(s="function"==typeof a?a(e):p(p({},e),a)),s},g=function(a){var e=N(a.components);return m.createElement(l.Provider,{value:e},a.children)},y="mdxType",i={inlineCode:"code",wrapper:function(a){var e=a.children;return m.createElement(m.Fragment,{},e)}},c=m.forwardRef((function(a,e){var s=a.components,n=a.mdxType,t=a.originalType,l=a.parentName,g=r(a,["components","mdxType","originalType","parentName"]),y=N(s),c=n,o=y["".concat(l,".").concat(c)]||y[c]||i[c]||t;return s?m.createElement(o,p(p({ref:e},g),{},{components:s})):m.createElement(o,p({ref:e},g))}));function o(a,e){var s=arguments,n=e&&e.mdxType;if("string"==typeof a||n){var t=s.length,p=new Array(t);p[0]=c;var r={};for(var l in e)hasOwnProperty.call(e,l)&&(r[l]=e[l]);r.originalType=a,r[y]="string"==typeof a?a:n,p[1]=r;for(var N=2;N<t;N++)p[N]=s[N];return m.createElement.apply(null,p)}return m.createElement.apply(null,s)}c.displayName="MDXCreateElement"},3194:(a,e,s)=>{s.r(e),s.d(e,{assets:()=>g,contentTitle:()=>l,default:()=>o,frontMatter:()=>r,metadata:()=>N,toc:()=>y});var m=s(58168),n=s(98587),t=(s(96540),s(15680));const p=["components"],r={},l="Activation Functions",N={unversionedId:"MachineLearning/NeuralNetworks/ActivationFunctions",id:"MachineLearning/NeuralNetworks/ActivationFunctions",title:"Activation Functions",description:"Activation functions are a fundamental component of neural networks that introduce non-linearity into the model, enabling neural networks to learn complex patterns and relationships in data.",source:"@site/docs/MachineLearning/NeuralNetworks/ActivationFunctions.md",sourceDirName:"MachineLearning/NeuralNetworks",slug:"/MachineLearning/NeuralNetworks/ActivationFunctions",permalink:"/MachineLearning/NeuralNetworks/ActivationFunctions",draft:!1,tags:[],version:"current",frontMatter:{},sidebar:"MachineLearning",previous:{title:"Neural Networks",permalink:"/category/neural-networks"},next:{title:"Reinforcement Learning",permalink:"/category/reinforcement-learning"}},g={},y=[{value:"Why Do We Need Activation Functions?",id:"why-do-we-need-activation-functions",level:2},{value:"Non-Linearity is Essential",id:"non-linearity-is-essential",level:3},{value:"Differentiability for Gradient-Based Optimization",id:"differentiability-for-gradient-based-optimization",level:3},{value:"Common Activation Functions",id:"common-activation-functions",level:2},{value:"Detailed Analysis",id:"detailed-analysis",level:2},{value:"Sigmoid Function",id:"sigmoid-function",level:3},{value:"Tanh (Hyperbolic Tangent)",id:"tanh-hyperbolic-tangent",level:3},{value:"ReLU (Rectified Linear Unit)",id:"relu-rectified-linear-unit",level:3},{value:"Leaky ReLU",id:"leaky-relu",level:3},{value:"ELU (Exponential Linear Unit)",id:"elu-exponential-linear-unit",level:3},{value:"Softmax",id:"softmax",level:3},{value:"Choosing Activation Functions",id:"choosing-activation-functions",level:2},{value:"For Hidden Layers",id:"for-hidden-layers",level:3},{value:"For Output Layer",id:"for-output-layer",level:3},{value:"Modern Trends",id:"modern-trends",level:2},{value:"GELU (Gaussian Error Linear Unit)",id:"gelu-gaussian-error-linear-unit",level:3},{value:"Swish (SiLU)",id:"swish-silu",level:3},{value:"Summary",id:"summary",level:2}],i={toc:y},c="wrapper";function o(a){let e=a.components,s=(0,n.A)(a,p);return(0,t.yg)(c,(0,m.A)({},i,s,{components:e,mdxType:"MDXLayout"}),(0,t.yg)("h1",{id:"activation-functions"},"Activation Functions"),(0,t.yg)("p",null,"Activation functions are a fundamental component of neural networks that introduce non-linearity into the model, enabling neural networks to learn complex patterns and relationships in data."),(0,t.yg)("h2",{id:"why-do-we-need-activation-functions"},"Why Do We Need Activation Functions?"),(0,t.yg)("h3",{id:"non-linearity-is-essential"},"Non-Linearity is Essential"),(0,t.yg)("p",null,"If activation functions were linear (or absent), stacking multiple layers would still result in a linear transformation:"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"}
1,(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"f"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"W"),(0,t.yg)("mn",{parentName:"msub"},"2")),(0,t.yg)("mo",{parentName:"mrow"},"\u22c5"),(0,t.yg)("mi",{parentName:"mrow"},"f"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"W"),(0,t.yg)("mn",{parentName:"msub"},"1")),(0,t.yg)("mo",{parentName:"mrow"},"\u22c5"),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"f"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"W"),(0,t.yg)("mn",{parentName:"msub"},"2")),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"W"),(0,t.yg)("mn",{parentName:"msub"},"1")),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"W"),(0,t.yg)("mrow",{parentName:"msub"},(0,t.yg)("mi",{parentName:"mrow"},"c"),(0,t.yg)("mi",{parentName:"mrow"},"o"),(0,t.yg)("mi",{parentName:"mrow"},"m"),(0,t.yg)("mi",{parentName:"mrow"},"b"),(0,t.yg)("mi",{parentName:"mrow"},"i"),(0,t.yg)("mi",{parentName:"mrow"},"n"),(0,t.yg)("mi",{parentName:"mrow"},"e"),(0,t.yg)("mi",{parentName:"mrow"},"d"))),(0,t.yg)("mi",{parentName:"mrow"},"x")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"f(W_2 \\cdot f(W_1 \\cdot x)) = f(W_2 W_1 x) = W_{combined} x")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.10764em"}},"f"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.13889em"}},"W"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3011em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"2")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u22c5"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.10764em"}},"f"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.13889em"}},"W"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3011em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"1")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u22c5"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},"))"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.10764em"}},"f"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.13889em"}},"W"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3011em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"2")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.13889em"}},"W"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3011em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"1")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.8333em",verticalAlign:"-0.15em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.13889em"}},"W"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3361em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.1389em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"co"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"mbin"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"d"))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x")))))),(0,t.yg)("p",null,"This would make deep networks no more powerful than a single-layer network. ",(0,t.yg)("strong",{parentName:"p"},"Non-linearity allows neural networks to learn complex patterns, decision boundaries, and hierarchical representations.")),(0,t.yg)("h3",{id:"differentiability-for-gradient-based-optimization"},"Differentiability for Gradient-Based Optimization"),(0,t.yg)("p",null,"For gradient-based optimization (like gradient descent), activation functions must be differentiable. The gradient of the loss with respect to weights requires taking derivatives through the activation functions (via backpropagation):"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"},"J")),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"},"\u03b8"))),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"},"J")),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"},"h"))),(0,t.yg)("mo",{parentName:"mrow"},"\u22c5"),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"},"h")),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mi",{parentName:"mrow"}
1,"\u03b8")))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\frac{\\partial J}{\\partial \\theta} = \\frac{\\partial J}{\\partial h} \\cdot \\frac{\\partial h}{\\partial \\theta}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.0574em",verticalAlign:"-0.686em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.3714em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.314em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.02778em"}}
1,"\u03b8"))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.09618em"}},"J")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.686em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"})),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.0574em",verticalAlign:"-0.686em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.3714em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.314em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"h"))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.09618em"}},"J")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.686em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"})),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u22c5"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.0574em",verticalAlign:"-0.686em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.3714em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.314em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.02778em"}},"\u03b8"))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"h")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.686em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("p",null,"If the activation function is not differentiable, we cannot compute these gradients for backpropagation."),(0,t.yg)("h2",{id:"common-activation-functions"},"Common Activation Functions"),(0,t.yg)("table",null,(0,t.yg)("thead",{parentName:"table"},(0,t.yg)("tr",{parentName:"thead"},(0,t.yg)("th",{parentName:"tr",align:"left"},"Activation"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Formula"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Derivative"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Range"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Properties"))),(0,t.yg)("tbody",{parentName:"table"},(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Sigmoid")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mn",{parentName:"mfrac"},"1"),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow"},"+"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x")))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\sigma(x) = \\frac{1}{1 + e^{-x}}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.2484em",verticalAlign:"-0.4033em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8451em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.655em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"1"),(0,t.yg)("span",{parentName:"span",className:"mbin mtight"},"+"),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7027em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.786em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"}
1,"x")))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.394em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"1"))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.4033em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"\u03c3"),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\sigma'(x) = \\sigma(x)(1 - \\sigma(x))")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0019em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7519em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"1"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},"))")))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"(0,1)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Output interpretable as probability, prone to vanishing gradients")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Tanh")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"tanh"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x")))),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"+"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x")))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\tanh(x) = \\frac{e^x - e^{-x}}{e^x + e^{-x}}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"tanh"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.3907em",verticalAlign:"-0.4033em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.9874em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.655em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.5935em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.786em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mbin mtight"},"+"),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7027em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.786em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.394em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7385em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.931em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mbin mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8477em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.931em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"))))))))))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.4033em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mi",{parentName:"mrow"},"tanh"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061")),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mi",{parentName:"mrow"},"tanh"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061")),(0,t.yg)("mn",{parentName:"msup"},"2")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\tanh'(x) = 1 - \\tanh^2(x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0862em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},(0,t.yg)("span",{parentName:"span",className:"mop"},"tanh"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8362em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.1473em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.7278em",verticalAlign:"-0.0833em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"1"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.1484em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},(0,t.yg)("span",{parentName:"span",className:"mop"},"tanh"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8984em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.1473em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"2")))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"(-1,1)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Zero-centered, still suffers from vanishing gradients")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"ReLU")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"ReLU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"max"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"0"),(0,t.yg)("mo",{parentName:"mrow",separator:"true"},","),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ReLU}(x) = \\max(0, x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ReLU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"max"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"),(0,t.yg)("span",{parentName:"span",className:"mpunct"},","),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.1667em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mtext",{parentName:"msup"},"ReLU"),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mn",{parentName:"mstyle"},"1"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mn",{parentName:"mstyle"},"0"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ReLU}'(x) = \\begin{cases} 1 & x > 0 \\\\ 0 & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0751em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ReLU")),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8251em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.1362em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{top:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"1"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"[0,\u221e)"),(0,t.yg)("td",{parentName:"tr",align:"left"},'Most popular, computationally efficient, can suffer from "dying ReLU"')),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Leaky ReLU")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"LeakyReLU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"max"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"0.01"),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",separator:"true"},","),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{LeakyReLU}(x) = \\max(0.01x, x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"LeakyReLU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"max"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"0.01"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mpunct"},","),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.1667em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mtext",{parentName:"msup"},"LeakyReLU"),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mn",{parentName:"mstyle"},"1"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mn",{parentName:"mstyle"},"0.01"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{LeakyReLU}
1'(x) = \\begin{cases} 1 & x > 0 \\\\ 0.01 & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0862em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"LeakyReLU")),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8362em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.1473em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{top:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"1"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"0.01")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"(-\u221e,\u221e)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Prevents dying ReLU problem")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"ELU")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"ELU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mi",{parentName:"mstyle"},"x"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"\u03b1"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ELU}(x) = \\begin{cases} x & x > 0 \\\\ \\alpha(e^x - 1) & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ELU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{to
1p:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.0037em"}},"\u03b1"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6644em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"1"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mtext",{parentName:"msup"},"ELU"),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mn",{parentName:"mstyle"},"1"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mtext",{parentName:"mrow"},"ELU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"+"),(0,t.yg)("mi",{parentName:"mrow"},"\u03b1")))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ELU}'(x) = \\begin{cases} 1 & x > 0 \\\\ \\text{ELU}(x) + \\alpha & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0751em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ELU")),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8251em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.1362em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{top:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"1"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ELU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"+"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.0037em"}},"\u03b1")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"(-\u03b1,\u221e)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Smooth, self-normalizing, can be slow to compute")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Softmax")),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"softmax"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("msup",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("msub",{parentName:"msup"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i"))),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"msub"},"\u2211"),(0,t.yg)("mi",{parentName:"msub"},"j")),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("msub",{parentName:"msup"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"j")))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{softmax}(x_i) = \\frac{e^{x_i}}{\\sum_j e^{x_j}}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"softmax")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3117em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.5885em",verticalAlign:"-0.6775em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.911em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.6447em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mop mtight"},(0,t.yg)("span",{parentName:"span",className:"mop op-symbol small-op mtight",style:{position:"relative",top:"0em"}},"\u2211"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.1496em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.1786em",marginLeft:"0em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.4603em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mspace mtight",style:{marginRight:"0.1952em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.779em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.9714em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3448em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.3448em",marginLeft:"0em",marginRight:"0.1em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.6595em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j"))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.5092em"}},(0,t.yg)("span",{parentName:"span"}
1))))))))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.394em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7385em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.931em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3448em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.3448em",marginLeft:"0em",marginRight:"0.1em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.6595em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i"))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3147em"}},(0,t.yg)("span",{parentName:"span"})))))))))))))))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6775em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("span",{parentName:"td",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("mtext",{parentName:"mrow"},"softmax"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u2202"),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"j")))),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mtext",{parentName:"mrow"},"softmax"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"\u03b4"),(0,t.yg)("mrow",{parentName:"msub"},(0,t.yg)("mi",{parentName:"mrow"},"i"),(0,t.yg)("mi",{parentName:"mrow"},"j"))),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mtext",{parentName:"mrow"},"softmax"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"j")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\frac{\\partial \\text{softmax}(x_i)}{\\partial x_j} = \\text{softmax}(x_i)(\\delta_{ij} - \\text{softmax}(x_j))")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.5523em",verticalAlign:"-0.5423em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.01em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.655em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3281em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.357em",marginLeft:"0em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.2819em"}},(0,t.yg)("span",{parentName:"span"}
1))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.485em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight",style:{marginRight:"0.05556em"}},"\u2202"),(0,t.yg)("span",{parentName:"span",className:"mord text mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"softmax")),(0,t.yg)("span",{parentName:"span",className:"mopen mtight"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3281em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.357em",marginLeft:"0em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.143em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mclose mtight"},")"))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.5423em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"})),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0361em",verticalAlign:"-0.2861em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"softmax")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3117em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03785em"}},"\u03b4"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3117em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"-0.0379em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"ij"))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.2861em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0361em",verticalAlign:"-0.2861em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"softmax")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3117em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.55em",marginLeft:"0em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.2861em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mclose"},"))")))))),(0,t.yg)("td",{parentName:"tr",align:"left"},"(0,1)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Used for multi-class classification (output layer)")))),(0,t.yg)("admonition",{type:"note"},(0,t.yg)("p",{parentName:"admonition"},"ReLU is technically not differentiable at ",(0,t.yg)("span",{parentName:"p",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mn",{parentName:"mrow"},"0")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"x=0")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.4306em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.6444em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))))),", but in practice we define ",(0,t.yg)("span",{parentName:"p",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mtext",{parentName:"msup"},"ReLU"),(0,t.yg)("mo",{parentName:"msup",mathvariant:"normal",lspace:"0em",rspace:"0em"},"\u2032")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"0"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mn",{parentName:"mrow"},"0")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ReLU}'(0) = 0")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1.0751em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ReLU")),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8251em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.1362em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2032"))))))))),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.6444em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))))," (or 1), which works well.")),(0,t.yg)("h2",{id:"detailed-analysis"},"Detailed Analysis"),(0,t.yg)("h3",{id:"sigmoid-function"},"Sigmoid Function"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mn",{parentName:"mfrac"},"1"),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow"},"+"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x")))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\sigma(x) = \\frac{1}{1 + e^{-x}}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.0908em",verticalAlign:"-0.7693em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.3214em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.314em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"1"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"+"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6973em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.989em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"}
1,"x"))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"1")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7693em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Pros:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Output range (0,1) can be interpreted as probabilities"),(0,t.yg)("li",{parentName:"ul"},"Smooth gradient"),(0,t.yg)("li",{parentName:"ul"},"Historically important")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Cons:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Vanishing gradient problem"),": For very large or small inputs, gradient becomes very small (\u22480), slowing learning"),(0,t.yg)("li",{parentName:"ul"},"Not zero-centered: Can cause zig-zagging dynamics during gradient descent"),(0,t.yg)("li",{parentName:"ul"},"Computationally expensive (exponential)")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Use cases:")," Binary classification (output layer), gates in LSTM/GRU"),(0,t.yg)("h3",{id:"tanh-hyperbolic-tangent"},"Tanh (Hyperbolic Tangent)"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"tanh"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x")))),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"+"),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mrow",{parentName:"msup"},(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mi",{parentName:"mrow"},"x"))))),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mn",{parentName:"mrow"},"2"),(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"2"),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mn",{parentName:"mrow"},"1")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\tanh(x) = \\frac{e^x - e^{-x}}{e^x + e^{-x}} = 2\\sigma(2x) - 1")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"tanh"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.2177em",verticalAlign:"-0.7693em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.4483em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.314em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.5904em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.989em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"+"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6973em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.989em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6644em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7713em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.7693em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"})),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"2"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"2"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.6444em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"1")))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Pros:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Zero-centered output (better than sigmoid)"),(0,t.yg)("li",{parentName:"ul"},"Smooth gradient")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Cons:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Still suffers from vanishing gradient problem"),(0,t.yg)("li",{parentName:"ul"},"Computationally expensive")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Use cases:")," Hidden layers in RNNs, gates in LSTM/GRU"),(0,t.yg)("h3",{id:"relu-rectified-linear-unit"},"ReLU (Rectified Linear Unit)"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"ReLU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"max"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"0"),(0,t.yg)("mo",{parentName:"mrow",separator:"true"},","),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ReLU}(x) = \\max(0, x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ReLU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"max"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"),(0,t.yg)("span",{parentName:"span",className:"mpunct"},","),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.1667em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Pros:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"}
1,"Very computationally efficient"),": Simple max operation"),(0,t.yg)("li",{parentName:"ul"},"Helps mitigate vanishing gradient problem"),(0,t.yg)("li",{parentName:"ul"},"Encourages sparse activations (many neurons output 0)"),(0,t.yg)("li",{parentName:"ul"},"Empirically shown to accelerate convergence")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Cons:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Dying ReLU problem"),': Neurons can permanently "die" (always output 0) if they get stuck during training'),(0,t.yg)("li",{parentName:"ul"},"Not zero-centered"),(0,t.yg)("li",{parentName:"ul"},"Unbounded output")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Use cases:")," Default choice for hidden layers in feedforward and convolutional neural networks"),(0,t.yg)("h3",{id:"leaky-relu"},"Leaky ReLU"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"LeakyReLU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"max"),(0,t.yg)("mo",{parentName:"mrow"},"\u2061"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mn",{parentName:"mrow"},"0.01"),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",separator:"true"},","),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mi",{parentName:"mstyle"},"x"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mn",{parentName:"mrow"},"0.01"),(0,t.yg)("mi",{parentName:"mrow"},"x")))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{LeakyReLU}(x) = \\max(0.01x, x) = \\begin{cases} x & x > 0 \\\\ 0.01x & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"LeakyReLU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mop"},"max"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},"0.01"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mpunct"},","),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.1667em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{to
1p:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},"0.01"),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Pros:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Fixes the dying ReLU problem (negative inputs have small non-zero gradient)"),(0,t.yg)("li",{parentName:"ul"},"Computationally efficient")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Cons:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"The slope for negative values (0.01) is a hyperparameter"),(0,t.yg)("li",{parentName:"ul"},"Not always better than ReLU in practice")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Variants:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Parametric ReLU (PReLU)"),": Learns the slope parameter during training"),(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Randomized Leaky ReLU (RReLU)"),": Randomizes the slope during training")),(0,t.yg)("h3",{id:"elu-exponential-linear-unit"},"ELU (Exponential Linear Unit)"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"ELU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mrow",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"mrow",fence:"true"},"{"),(0,t.yg)("mtable",{parentName:"mrow",rowspacing:"0.36em",columnalign:"left left",columnspacing:"1em"},(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mi",{parentName:"mstyle"},"x"))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},">"),(0,t.yg)("mn",{parentName:"mrow"},"0"))))),(0,t.yg)("mtr",{parentName:"mtable"},(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"\u03b1"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("mi",{parentName:"msup"},"x")),(0,t.yg)("mo",{parentName:"mrow"},"\u2212"),(0,t.yg)("mn",{parentName:"mrow"},"1"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")))),(0,t.yg)("mtd",{parentName:"mtr"},(0,t.yg)("mstyle",{parentName:"mtd",scriptlevel:"0",displaystyle:"false"},(0,t.yg)("mrow",{parentName:"mstyle"},(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u2264"),(0,t.yg)("mn",{parentName:"mrow"},"0")))))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{ELU}(x) = \\begin{cases} x & x > 0 \\\\ \\alpha(e^x - 1) & x \\leq 0 \\end{cases}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"ELU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"3em",verticalAlign:"-1.25em"}}),(0,t.yg)("span",{parentName:"span",className:"minner"},(0,t.yg)("span",{parentName:"span",className:"mopen delimcenter",style:{to
1p:"0em"}},(0,t.yg)("span",{parentName:"span",className:"delimsizing size4"},"{")),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mtable"},(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.0037em"}},"\u03b1"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6644em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x")))))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u2212"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"1"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"arraycolsep",style:{width:"1em"}}),(0,t.yg)("span",{parentName:"span",className:"col-align-l"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.69em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.69em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},">"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0"))),(0,t.yg)("span",{parentName:"span",style:{top:"-2.25em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3.008em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"\u2264"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"0")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.19em"}},(0,t.yg)("span",{parentName:"span"}))))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Pros:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Smooth everywhere (including at 0)"),(0,t.yg)("li",{parentName:"ul"},"Negative saturation helps make network more robust to noise"),(0,t.yg)("li",{parentName:"ul"},"Closer to zero-mean outputs"),(0,t.yg)("li",{parentName:"ul"},"Can lead to faster learning")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Cons:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Computationally more expensive due to exponential"),(0,t.yg)("li",{parentName:"ul"},"Requires tuning of ",(0,t.yg)("span",{parentName:"li",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow"},"\u03b1")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\alpha")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.4306em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.0037em"}},"\u03b1")))))," hyperparameter")),(0,t.yg)("h3",{id:"softmax"},"Softmax"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"softmax"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("msub",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i")),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mfrac",{parentName:"mrow"},(0,t.yg)("msup",{parentName:"mfrac"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("msub",{parentName:"msup"},(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"i"))),(0,t.yg)("mrow",{parentName:"mfrac"},(0,t.yg)("munderover",{parentName:"mrow"},(0,t.yg)("mo",{parentName:"munderover"},"\u2211"),(0,t.yg)("mrow",{parentName:"munderover"},(0,t.yg)("mi",{parentName:"mrow"},"j"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mn",{parentName:"mrow"},"1")),(0,t.yg)("mi",{parentName:"munderover"},"n")),(0,t.yg)("msup",{parentName:"mrow"},(0,t.yg)("mi",{parentName:"msup"},"e"),(0,t.yg)("msub",{parentName:"msup"}
1,(0,t.yg)("mi",{parentName:"msub"},"x"),(0,t.yg)("mi",{parentName:"msub"},"j")))))),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{softmax}(x_i) = \\frac{e^{x_i}}{\\sum_{j=1}^{n} e^{x_j}}")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"softmax")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3117em"}},(0,t.yg)("span",{parentName:"span",style:{to
1p:"-2.55em",marginLeft:"0em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.15em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"2.4715em",verticalAlign:"-1.1301em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mopen nulldelimiter"}),(0,t.yg)("span",{parentName:"span",className:"mfrac"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.3414em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.3057em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mop"},(0,t.yg)("span",{parentName:"span",className:"mop op-symbol small-op",style:{position:"relative",top:"0em"}},"\u2211"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.8043em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.4003em",marginLeft:"0em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j"),(0,t.yg)("span",{parentName:"span",className:"mrel mtight"},"="),(0,t.yg)("span",{parentName:"span",className:"mord mtight"}
1,"1")))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.2029em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"n"))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.4358em"}},(0,t.yg)("span",{parentName:"span"})))))),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.1667em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6065em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-3.0051em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3281em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-2.357em",marginLeft:"0em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight",style:{marginRight:"0.05724em"}},"j")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.2819em"}},(0,t.yg)("span",{parentName:"span"}
1)))))))))))))))),(0,t.yg)("span",{parentName:"span",style:{top:"-3.23em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"frac-line",style:{borderBottomWidth:"0.04em"}})),(0,t.yg)("span",{parentName:"span",style:{top:"-3.677em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"3em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"e"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.6644em"}},(0,t.yg)("span",{parentName:"span",style:{top:"-3.063em",marginRight:"0.05em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.7em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size6 size3 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"x"),(0,t.yg)("span",{parentName:"span",className:"msupsub"},(0,t.yg)("span",{parentName:"span",className:"vlist-t vlist-t2"},(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.3281em"}}
1,(0,t.yg)("span",{parentName:"span",style:{top:"-2.357em",marginLeft:"0em",marginRight:"0.0714em"}},(0,t.yg)("span",{parentName:"span",className:"pstrut",style:{height:"2.5em"}}),(0,t.yg)("span",{parentName:"span",className:"sizing reset-size3 size1 mtight"},(0,t.yg)("span",{parentName:"span",className:"mord mathnormal mtight"},"i")))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"0.143em"}},(0,t.yg)("span",{parentName:"span"}))))))))))))))))),(0,t.yg)("span",{parentName:"span",className:"vlist-s"},"\u200b")),(0,t.yg)("span",{parentName:"span",className:"vlist-r"},(0,t.yg)("span",{parentName:"span",className:"vlist",style:{height:"1.1301em"}},(0,t.yg)("span",{parentName:"span"}))))),(0,t.yg)("span",{parentName:"span",className:"mclose nulldelimiter"}))))))),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Properties:")),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},"Outputs sum to 1 (can be interpreted as probability distribution)"),(0,t.yg)("li",{parentName:"ul"},"Differentiable"),(0,t.yg)("li",{parentName:"ul"},"Sensitive to outliers (large values dominate)")),(0,t.yg)("p",null,(0,t.yg)("strong",{parentName:"p"},"Use cases:")," Multi-class classification (output layer), attention mechanisms"),(0,t.yg)("h2",{id:"choosing-activation-functions"},"Choosing Activation Functions"),(0,t.yg)("h3",{id:"for-hidden-layers"},"For Hidden Layers"),(0,t.yg)("ol",null,(0,t.yg)("li",{parentName:"ol"},(0,t.yg)("strong",{parentName:"li"},"Start with ReLU")," - It's the default choice and works well in most cases"),(0,t.yg)("li",{parentName:"ol"},(0,t.yg)("strong",{parentName:"li"},"Try Leaky ReLU / ELU")," - If you encounter dying ReLU problem"),(0,t.yg)("li",{parentName:"ol"},(0,t.yg)("strong",{parentName:"li"},"Use Tanh")," - For RNNs and when you need zero-centered outputs"),(0,t.yg)("li",{parentName:"ol"},(0,t.yg)("strong",{parentName:"li"},"Avoid Sigmoid")," - In hidden layers due to vanishing gradient problem")),(0,t.yg)("h3",{id:"for-output-layer"},"For Output Layer"),(0,t.yg)("p",null,"The choice depends on your task:"),(0,t.yg)("table",null,(0,t.yg)("thead",{parentName:"table"},(0,t.yg)("tr",{parentName:"thead"},(0,t.yg)("th",{parentName:"tr",align:"left"},"Task"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Activation"),(0,t.yg)("th",{parentName:"tr",align:"left"},"Reason"))),(0,t.yg)("tbody",{parentName:"table"},(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Binary Classification")),(0,t.yg)("td",{parentName:"tr",align:"left"},"Sigmoid"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Outputs probability between 0 and 1")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Multi-class Classification")),(0,t.yg)("td",{parentName:"tr",align:"left"},"Softmax"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Outputs probability distribution over classes")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Regression")),(0,t.yg)("td",{parentName:"tr",align:"left"},"Linear (None)"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Unrestricted output range")),(0,t.yg)("tr",{parentName:"tbody"},(0,t.yg)("td",{parentName:"tr",align:"left"},(0,t.yg)("strong",{parentName:"td"},"Positive Regression")),(0,t.yg)("td",{parentName:"tr",align:"left"},"ReLU / Softplus"),(0,t.yg)("td",{parentName:"tr",align:"left"},"Ensures positive outputs")))),(0,t.yg)("h2",{id:"modern-trends"},"Modern Trends"),(0,t.yg)("h3",{id:"gelu-gaussian-error-linear-unit"},"GELU (Gaussian Error Linear Unit)"),(0,t.yg)("p",null,"Used in transformers (BERT, GPT):"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"}
1,(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"GELU"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u22c5"),(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u03a6"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{GELU}(x) = x \\cdot \\Phi(x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"GELU")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.4445em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u22c5"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"\u03a6"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("p",null,"Where ",(0,t.yg)("span",{parentName:"p",className:"math math-inline"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mi",{parentName:"mrow",mathvariant:"normal"},"\u03a6"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\Phi(x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord"},"\u03a6"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))," is the cumulative distribution function of the standard normal distribution."),(0,t.yg)("h3",{id:"swish-silu"},"Swish (SiLU)"),(0,t.yg)("p",null,"Self-gated activation:"),(0,t.yg)("div",{className:"math math-display"},(0,t.yg)("span",{parentName:"div",className:"katex-display"},(0,t.yg)("span",{parentName:"span",className:"katex"},(0,t.yg)("span",{parentName:"span",className:"katex-mathml"},(0,t.yg)("math",{parentName:"span",xmlns:"http://www.w3.org/1998/Math/MathML",display:"block"},(0,t.yg)("semantics",{parentName:"math"},(0,t.yg)("mrow",{parentName:"semantics"},(0,t.yg)("mtext",{parentName:"mrow"},"Swish"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")"),(0,t.yg)("mo",{parentName:"mrow"},"="),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow"},"\u22c5"),(0,t.yg)("mi",{parentName:"mrow"},"\u03c3"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},"("),(0,t.yg)("mi",{parentName:"mrow"},"x"),(0,t.yg)("mo",{parentName:"mrow",stretchy:"false"},")")),(0,t.yg)("annotation",{parentName:"semantics",encoding:"application/x-tex"},"\\text{Swish}(x) = x \\cdot \\sigma(x)")))),(0,t.yg)("span",{parentName:"span",className:"katex-html","aria-hidden":"true"},(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord text"},(0,t.yg)("span",{parentName:"span",className:"mord"},"Swish")),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}}),(0,t.yg)("span",{parentName:"span",className:"mrel"},"="),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2778em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"0.4445em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}}),(0,t.yg)("span",{parentName:"span",className:"mbin"},"\u22c5"),(0,t.yg)("span",{parentName:"span",className:"mspace",style:{marginRight:"0.2222em"}})),(0,t.yg)("span",{parentName:"span",className:"base"},(0,t.yg)("span",{parentName:"span",className:"strut",style:{height:"1em",verticalAlign:"-0.25em"}}),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal",style:{marginRight:"0.03588em"}},"\u03c3"),(0,t.yg)("span",{parentName:"span",className:"mopen"},"("),(0,t.yg)("span",{parentName:"span",className:"mord mathnormal"},"x"),(0,t.yg)("span",{parentName:"span",className:"mclose"},")")))))),(0,t.yg)("p",null,"Shown to outperform ReLU in deep networks."),(0,t.yg)("h2",{id:"summary"},"Summary"),(0,t.yg)("ul",null,(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Activation functions introduce non-linearity"),", essential for neural networks to learn complex patterns"),(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"ReLU is the default choice")," for hidden layers in most architectures"),(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Sigmoid and Softmax")," are used in output layers for classification"),(0,t.yg)("li",{parentName:"ul"},(0,t.yg)("strong",{parentName:"li"},"Modern alternatives")," (GELU, Swish) show promise in specific architectures"),(0,t.yg)("li",{parentName:"ul"},"Choice depends on architecture, task, and empirical performance")))}o.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.