1 2<!DOCTYPE html> 3 4 5<html lang="en" data-content_root="../" data-theme="light"> 6 7 <head> 8 <meta charset="utf-8" /> 9 <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" /> 10 11 <title>Gradient Descent — Machine Learning Visualized</title> 12 13 14 15
15<script data-cfasync="false"> 16 document.documentElement.dataset.mode = localStorage.getItem("mode") || "light"; 17 document.documentElement.dataset.theme = localStorage.getItem("theme") || "light"; 18 </script>
18 19 20 <!-- Loaded before other Sphinx assets --> 21 <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> 22<link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> 23<link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> 24 25 26 <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" /> 27 <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" /> 28<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" /> 29<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" /> 30 31 <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" /> 32 <link rel="stylesheet" type="text/css" href="../_static/styles/sphinx-book-theme.css?v=a3416100" /> 33 <link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=9c3e77be" /> 34 <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" /> 35 <link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" /> 36 <link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css?v=4fa983c6" /> 37 <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" /> 38 <link rel="stylesheet" type="text/css" href="../_static/custom.css?v=3a394b7d" /> 39 40 <!-- Pre-loaded scripts that we'll load fully later --> 41 <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" /> 42<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" /> 43
43<script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script>
43 44 45
45<script src="../_static/documentation_options.js?v=9eb32ce0"></script>
45 46
46<script src="../_static/doctools.js?v=9a2dae69"></script>
46 47
47<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
47 48
48<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
48 49
49<script src="../_static/copybutton.js?v=f281be69"></script>
49 50
50<script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
50 51
51<script>let toggleHintShow = 'Click to show';</script>
51 52
52<script>let toggleHintHide = 'Click to hide';</script>
52 53
53<script>let toggleOpenOnPrint = 'true';</script>
53 54
54<script src="../_static/togglebutton.js?v=b8b12719"></script>
54 55
55<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
55 56
56<script src="../_static/design-tabs.js?v=f930bc37"></script>
vendor: 76 bytes, lines 56-57
56 57 <script async="async" src="https://www.googletagmanager.com/gtag/js?id=
57G-9S2CQ3B60T
vendor: 16 bytes, lines 57-58
57"></script> 58
58<script> 59
vendor: 177 bytes, lines 59-62
59window.dataLayer = window.dataLayer || []; 60 function gtag(){ dataLayer.push(arguments); } 61 gtag('js', new Date()); 62 gtag('config', '
62G-9S2CQ3B60T
vendor: 16 bytes, lines 62-63
62'); 63
63</script>
63 64
64<script>const THEBE_JS_URL = "https://unpkg.com/[email protected]/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
64 65
65<script async="async" src="../_static/sphinx-thebe.js?v=c100c467"></script>
65 66
66<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
66 67
67<script> 68
vendor: 177 bytes, lines 68-71
68window.dataLayer = window.dataLayer || []; 69 function gtag(){ dataLayer.push(arguments); } 70 gtag('js', new Date()); 71 gtag('config', '
71G-9S2CQ3B60T
vendor: 16 bytes, lines 71-72
71'); 72
72</script>
72 73
73<script>const THEBE_JS_URL = "https://unpkg.com/[email protected]/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
73 74
74<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
74 75
75<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
75 76
76<script>DOCUMENTATION_OPTIONS.pagename = 'chapter1/linear_regression';</script>
76 77 <link rel="icon" href="../_static/logo.png"/> 78 <link rel="index" title="Index" href="../genindex.html" /> 79 <link rel="search" title="Search" href="../search.html" /> 80 <link rel="next" title="Optimizers" href="optimizers.html" /> 81 <link rel="prev" title="Machine Learning Visualized" href="../index.html" /> 82 <meta name="viewport" content="width=device-width, initial-scale=1"/> 83 <meta name="docsearch:language" content="en"/> 84 </head> 85 86 87 <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light"> 88 89 90 91 <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a> 92 93 <div id="pst-scroll-pixel-helper"></div> 94 95 <button type="button" class="btn rounded-pill" id="pst-back-to-top"> 96 <i class="fa-solid fa-arrow-up"></i> 97 Back to top 98 </button> 99 100 101 <input type="checkbox" 102 class="sidebar-toggle" 103 name="__primary" 104 id="__primary"/> 105 <label class="overlay overlay-primary" for="__primary"></label> 106 107 <input type="checkbox" 108 class="sidebar-toggle" 109 name="__secondary" 110 id="__secondary"/> 111 <label class="overlay overlay-secondary" for="__secondary"></label> 112 113 <div class="search-button__wrapper"> 114 <div class="search-button__overlay"></div> 115 <div class="search-button__search-container"> 116<form class="bd-search d-flex align-items-center" 117 action="../search.html" 118 method="get"> 119 <i class="fa-solid fa-magnifying-glass"></i> 120 <input type="search" 121 class="form-control" 122 name="q" 123 id="search-input" 124 placeholder="Search this book..." 125 aria-label="Search this book..." 126 autocomplete="off" 127 autocorrect="off" 128 autocapitalize="off" 129 spellcheck="false"/> 130 <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span> 131</form></div> 132 </div> 133 134 <header class="bd-header navbar navbar-expand-lg bd-navbar"> 135 </header> 136 137 138 <div class="bd-container"> 139 <div class="bd-container__inner bd-page-width"> 140 141 142 143 <div class="bd-sidebar-primary bd-sidebar"> 144 145 146 147 <div class="sidebar-header-items sidebar-primary__section"> 148 149 150 151 152 </div> 153 154 <div class="sidebar-primary-items__start sidebar-primary__section"> 155 <div class="sidebar-primary-item"> 156 157 158 159<a class="navbar-brand logo" href="../index.html"> 160 161 162 163 164 165 166 <p class="title logo__title">Machine Learning Visualized</p> 167 168</a></div> 169 <div class="sidebar-primary-item"> 170 171
171<script> 172 document.write(` 173 <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> 174 <i class="fa-solid fa-magnifying-glass"></i> 175 <span class="search-button__default-text">Search</span> 176 <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span> 177 </button> 178 `); 179 </script>
179</div> 180 <div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main"> 181 <div class="bd-toc-item navbar-nav active"> 182 183 <ul class="nav bd-sidenav bd-sidenav__home-link"> 184 <li class="toctree-l1"> 185 <a class="reference internal" href="../index.html"> 186 Machine Learning Visualized 187 </a> 188 </li> 189 </ul> 190 <p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 1. Optimization</span></p> 191<ul class="current nav bd-sidenav"> 192<li class="toctree-l1 current active"><a class="current reference internal" href="#">Gradient Descent</a></li> 193<li class="toctree-l1"><a class="reference internal" href="optimizers.html">Optimizers</a></li> 194<li class="toctree-l1"><a class="reference internal" href="interactive_linear_regression.html">Interactive Linear Regression</a></li> 195</ul> 196<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 2. Clustering and Reduction</span></p> 197<ul class="nav bd-sidenav"> 198<li class="toctree-l1"><a class="reference internal" href="../chapter2/pca.html">Principal Component Analysis</a></li> 199<li class="toctree-l1"><a class="reference internal" href="../chapter2/k_means.html">K-Means Clustering</a></li> 200</ul> 201<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 3. Linear Models</span></p> 202<ul class="nav bd-sidenav"> 203<li class="toctree-l1"><a class="reference internal" href="../chapter3/perceptron.html">Perceptron</a></li> 204<li class="toctree-l1"><a class="reference internal" href="../chapter3/logistic_regression.html">Logistic Regression</a></li> 205<li class="toctree-l1"><a class="reference internal" href="../chapter3/interactive_perceptron.html">Interactive Perceptron</a></li> 206<li class="toctree-l1"><a class="reference internal" href="../chapter3/interactive_logistic_regression.html">Interactive Logistic Regression</a></li> 207</ul> 208<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 4. Neural Networks</span></p> 209<ul class="nav bd-sidenav"> 210<li class="toctree-l1"><a class="reference internal" href="../chapter4/neural_network.html">Neural Network</a></li> 211<li class="toctree-l1"><a class="reference internal" href="../chapter4/neural_network_weights.html">Neural Network Weights Visualization</a></li> 212<li class="toctree-l1"><a class="reference internal" href="../chapter4/autoencoder.html">Autoencoder</a></li> 213</ul> 214 215 </div> 216</nav></div> 217 </div> 218 219 220 <div class="sidebar-primary-items__end sidebar-primary__section"> 221 </div> 222 223 <div id="rtd-footer-container"></div> 224 225 226 </div> 227 228 <main id="main-content" class="bd-main"> 229 230 231 232<div class="sbt-scroll-pixel-helper"></div> 233 234 <div class="bd-content"> 235 <div class="bd-article-container"> 236 237 <div class="bd-header-article"> 238<div class="header-article-items header-article__inner"> 239 240 <div class="header-article-items__start"> 241 242 <div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip"> 243 <span class="fa-solid fa-bars"></span> 244</button></div> 245 246 </div> 247 248 249 <div class="header-article-items__end"> 250 251 <div class="header-article-item"> 252 253<div class="article-header-buttons"> 254 255 256 257 258 259<div class="dropdown dropdown-source-buttons"> 260 <button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Source repositories"> 261 <i class="fab fa-github"></i> 262 </button> 263 <ul class="dropdown-menu"> 264 265 266 267 <li><a href="https://github.com/gavinkhung/machine-learning-visualized" target="_blank" 268 class="btn btn-sm btn-source-repository-button dropdown-item" 269 title="Source repository" 270 data-bs-placement="left" data-bs-toggle="tooltip" 271> 272 273 274<span class="btn__icon-container"> 275 <i class="fab fa-github"></i> 276 </span> 277<span class="btn__text-container">Repository</span> 278</a> 279</li> 280 281 282 283 284 <li>
284<a href="https://github.com/gavinkhung/machine-learning-visualized/issues/new?title=Issue%20on%20page%20%2Fchapter1/linear_regression.html&body=Your%20issue%20content%20here." target="_blank" 285 class="btn btn-sm btn-source-issues-button dropdown-item" 286 title="Open an issue" 287 data-bs-placement="left" data-bs-toggle="tooltip" 288> 289 290 291<span class="btn__icon-container"> 292 <i class="fas fa-lightbulb"></i> 293 </span> 294<span class="btn__text-container">Open issue</span> 295</a> 296</li> 297 298 </ul> 299</div> 300 301 302 303 304 305 306<div class="dropdown dropdown-download-buttons"> 307 <button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page"> 308 <i class="fas fa-download"></i> 309 </button> 310 <ul class="dropdown-menu"> 311 312 313 314 <li><a href="../_sources/chapter1/linear_regression.ipynb" target="_blank" 315 class="btn btn-sm btn-download-source-button dropdown-item" 316 title="Download source file" 317 data-bs-placement="left" data-bs-toggle="tooltip" 318> 319 320 321<span class="btn__icon-container"> 322 <i class="fas fa-file"></i> 323 </span> 324<span class="btn__text-container">.ipynb</span> 325</a> 326</li> 327 328 329 330 331 <li> 332<button onclick="window.print()" 333 class="btn btn-sm btn-download-pdf-button dropdown-item" 334 title="Print to PDF" 335 data-bs-placement="left" data-bs-toggle="tooltip" 336> 337 338 339<span class="btn__icon-container"> 340 <i class="fas fa-file-pdf"></i> 341 </span> 342<span class="btn__text-container">.pdf</span> 343</button> 344</li> 345 346 </ul> 347</div> 348 349 350 351 352<button onclick="toggleFullScreen()" 353 class="btn btn-sm btn-fullscreen-button" 354 title="Fullscreen mode" 355 data-bs-placement="bottom" data-bs-toggle="tooltip" 356> 357 358 359<span class="btn__icon-container"> 360 <i class="fas fa-expand"></i> 361 </span> 362 363</button> 364 365 366
367<script> 368document.write(` 369 <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip"> 370 <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span> 371 <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span> 372 <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span> 373 </button> 374`); 375</script>
375 376 377
378<script> 379document.write(` 380 <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip"> 381 <i class="fa-solid fa-magnifying-glass fa-lg"></i> 382 </button> 383`); 384</script>
384 385<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip"> 386 <span class="fa-solid fa-list"></span> 387</button> 388</div></div> 389 390 </div> 391 392</div> 393</div> 394 395 396 397<div id="jb-print-docs-body" class="onlyprint"> 398 <h1>Gradient Descent</h1> 399 <!-- Table of contents --> 400 <div id="print-main-content"> 401 <div id="jb-print-toc"> 402 403 <div> 404 <h2> Contents </h2> 405 </div> 406 <nav aria-label="Page"> 407 <ul class="visible nav section-nav flex-column"> 408<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-dataset">Training Dataset</a></li> 409<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-mean-squared-error">Loss Function: Mean Squared Error</a></li> 410<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-gradient">Loss Function Gradient</a></li> 411<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-w">Loss Function in Terms of W</a></li> 412<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-b">Loss Function in Terms of b</a></li> 413<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-linear-regression-model">Training the Linear Regression Model</a></li> 414<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#graphing-functions">Graphing functions</a></li> 415<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-model">Training the model</a></li> 416</ul> 417 </nav> 418 </div> 419 </div> 420</div> 421 422 423 424<div id="searchbox"></div> 425 <article class="bd-article"> 426 427 <section class="tex2jax_ignore mathjax_ignore" id="gradient-descent"> 428<h1>Gradient Descent<a class="headerlink" href="#gradient-descent" title="Link to this heading">#</a></h1> 429<p>Gradient Descent is an algorithm that finds the local minimum of a function. This is applicable to machine learning because we want to find the optimal parameters that minimize our loss function. In machine learning, loss functions quantify the amount of error between the predicted values from a machine learning model and the actual expected values. In this notebook, we will perform linear regression by using gradient descent to find the optimal slope and y-intercept.</p> 430<section id="training-dataset"> 431<h2>Training Dataset<a class="headerlink" href="#training-dataset" title="Link to this heading">#</a></h2> 432<p>Importing the libraries</p> 433<div class="cell docutils container"> 434<div class="cell_input docutils container"> 435<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">np</span> 436<span class="kn">import</span><span class="w"> </span><span class="nn">matplotlib.pyplot</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">plt</span> 437<span class="kn">import</span><span class="w"> </span><span class="nn">scienceplots</span> 438<span class="kn">from</span><span class="w"> </span><span class="nn">IPython.display</span><span class="w"> </span><span class="kn">import</span> <span class="n">display</span><span class="p">,</span> <span class="n">Latex</span><span class="p">,</span> <span class="n">Image</span> 439<span class="kn">from</span><span class="w"> </span><span class="nn">celluloid</span><span class="w"> </span><span class="kn">import</span> <span class="n">Camera</span> 440 441<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> 442<span class="n">plt</span><span class="o">.</span><span class="n">style</span><span class="o">.</span><span class="n">
442use</span><span class="p">([</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"no-latex"</span><span class="p">])</span> 443</pre></div> 444</div> 445</div> 446</div> 447<p>Letâs look at the training dataset. We will use columns 2 and 4 of the txt file. The linear regression model will find the optimal slope and y-intercept to fit the data.</p> 448<div class="cell docutils container"> 449<div class="cell_input docutils container"> 450<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fname</span> <span class="o">=</span> <span class="s2">"REGRESSION-gradientDescent-data.txt"</span> 451<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">loadtxt</span><span class="p">(</span><span class="n">fname</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s2">","</span><span class="p">,</span> <span class="n">unpack</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">skiprows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">usecols</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span> 452 453<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span> 454<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">()</span> 455<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"o"</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span> 456</pre></div> 457</div> 458</div> 459<div class="cell_output docutils container"> 460<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><matplotlib.collections.PathCollection at 0x7a9104ff5e20> 461</pre></div> 462</div> 463<img alt="../_images/fa4746a5ae55598f4a4e28ae287ca7cc834b20a984c460cef12a8165c03f109e.png" src="../_images/fa4746a5ae55598f4a4e28ae287ca7cc834b20a984c460cef12a8165c03f109e.png" /> 464</div> 465</div> 466</section> 467<section id="loss-function-mean-squared-error"> 468<h2>Loss Function: Mean Squared Error<a class="headerlink" href="#loss-function-mean-squared-error" title="Link to this heading">#</a></h2> 469<p>For the linear regression model, the predicted value <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the product of the weight <span class="math notranslate nohighlight">\(w\)</span> and the input feature <span class="math notranslate nohighlight">\(x_i\)</span> plus a bias term <span class="math notranslate nohighlight">\(b\)</span>:</p> 470<div class="math notranslate nohighlight"> 471\[\hat{y}_i = w x_i + b\]</div> 472<p>We will use the mean squared error function as our loss function:</p> 473<div class="math notranslate nohighlight"> 474\[\begin{split} 475\begin{align*} 476MSE &= \frac{1}{n} \sum_{i=1}^{n}(y_{i}-\hat{y}_i)^2 \\ 477&= \frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2 478\end{align*} 479\end{split}\]</div> 480</section> 481<section id="loss-function-gradient"> 482<h2>Loss Function Gradient<a class="headerlink" href="#loss-function-gradient" title="Link to this heading">#</a></h2> 483<div class="cell docutils container"> 484<div class="cell_input docutils container"> 485<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span> 486 <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">)))</span> 487</pre></div> 488</div> 489</div> 490</div> 491<p>In each epoch of gradient descent, a parameter is updated by subtracting the product of the gradient of the function and the learning rate (<span class="math notranslate nohighlight">\(\eta\)</span>). The learning rate controls how much the parameters should change. Small learning rates are precise, but are slow. Large learning rates are fast, but may overshoot and prevent the model from finding the minimum.</p> 492<div class="math notranslate nohighlight"> 493\[ 494\begin{align*} 495X_{t+1} = X_t - \eta \nabla f(X_t) 496\end{align*} 497\]</div> 498<p>Since we are finding the optimal slope (<span class="math notranslate nohighlight">\(w\)</span>) and y-intercept (<span class="math notranslate nohighlight">\(b\)</span>) for our linear regression model, we must find the partial derivatives of the loss function with respect to <span class="math notranslate nohighlight">\(w\)</span> and <span class="math notranslate nohighlight">\(b\)</span>.</p> 499</section> 500<section id="loss-function-in-terms-of-w"> 501<h2>Loss Function in Terms of W<a class="headerlink" href="#loss-function-in-terms-of-w" title="Link to this heading">#</a></h2> 502<p>Loss function with respect to <span class="math notranslate nohighlight">\(w\)</span>:</p> 503<div class="math notranslate nohighlight"> 504\[\begin{split} 505\begin{align*} 506\frac{\partial}{\partial w} \left( MSE \right) &= \frac{\partial}{\partial w}\left[\frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2\right] \\ 507&= \frac{1}{n} \sum_{i=1}^{n} \frac{\partial}{\partial w}\left[(y_{i}-(w x_{i} + b))^2\right] \\ 508&= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\frac{\partial}{\partial w}\left[y_{i}-(w x_{i} + b)\right] \\ 509&= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))(-x_{i}) \\ 510&= -\frac{2}{n} \sum_{i=1}^{n} x_{i}(y_{i}-(w x_{i} + b)) 511\end{align*} 512\end{split}\]</div> 513<div class="cell docutils container">
514<div class="cell_input docutils container"> 515<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss_dw</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span> 516 <span class="k">return</span> <span class="o">-</span><span class="mi">2</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">)))</span> 517</pre></div> 518</div> 519</div> 520</div> 521</section> 522<section id="loss-function-in-terms-of-b"> 523<h2>Loss Function in Terms of b<a class="headerlink" href="#loss-function-in-terms-of-b" title="Link to this heading">#</a></h2> 524<p>Loss function with respect to <span class="math notranslate nohighlight">\(b\)</span>:</p> 525<div class="math notranslate nohighlight"> 526\[\begin{split} 527\begin{align*} 528\frac{\partial}{\partial b} \left( MSE \right) &= \frac{\partial}{\partial b}\left[\frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2\right] \\ 529&= \frac{1}{n} \sum_{i=1}^{n} \frac{\partial}{\partial b}\left[(y_{i}-(w x_{i} + b))^2\right] \\ 530&= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\frac{\partial}{\partial b}\left[y_{i}-(w x_{i} + b)\right] \\ 531&= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))(-1) \\ 532&= -\frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b)) 533\end{align*} 534\end{split}\]</div> 535<div class="cell docutils container"> 536<div class="cell_input docutils container"> 537<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss_db</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span> 538 <span class="k">return</span> <span class="o">-</span><span class="mi">2</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">))</span> 539</pre></div> 540</div> 541</div> 542</div> 543</section> 544<section id="training-the-linear-regression-model"> 545<h2>Training the Linear Regression Model<a class="headerlink" href="#training-the-linear-regression-model" title="Link to this heading">#</a></h2> 546<p>Letâs define a function that uses the gradient algorithm to update the parameters of the loss function. The function uses the gradient functions we derived earlier.</p> 547<p>General Gradient Descent Equation:</p> 548<div class="math notranslate nohighlight"> 549\[X_{t+1} = X_t - \eta \nabla f(X_t)\]</div> 550<p>Weights Gradient Descent:</p> 551<div class="math notranslate nohighlight"> 552\[\begin{split} 553\begin{align*} 554w &= w - \eta \frac{\partial}{\partial w} [L(w, b)] \\ 555&= w - \eta \left[-\frac{2}{n} \sum_{i=1}^{n} x_{i}(y_{i}-(w x_{i} + b))\right] 556\end{align*} 557\end{split}\]</div> 558<p>Bias Gradient Descent:</p> 559<div class="math notranslate nohighlight"> 560\[\begin{split} 561\begin{align*} 562b &= b - \eta \frac{\partial}{\partial b} [L(w, b)] \\ 563&= b - \eta \left[-\frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\right] 564\end{align*} 565\end{split}\]</div> 566<div class="cell docutils container"> 567<div class="cell_input docutils container"> 568<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">update_w_and_b</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">):</span> 569 <span class="n">dw</span> <span class="o">=</span> <span class="n">mse_loss_dw</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> 570 <span class="n">db</span> <span class="o">=</span> <span class="n">mse_loss_db</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> 571 572 <span class="n">w</span> <span class="o">=</span> <span class="n">w</span> <span class="o">-</span> <span class="n">dw</span> <span class="o">*</span> <span class="n">learning_rate</span> 573 <span class="n">b</span> <span class="o">=</span> <span class="n">b</span> <span class="o">-</span> <span class="n">db</span> <span class="o">*</span> <span class="n">learning_rate</span> 574 575 <span class="k">return</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span> 576</pre></div> 577</div> 578</div> 579</div> 580</section> 581<section id="graphing-functions"> 582<h2>Graphing functions<a class="headerlink" href="#graphing-functions" title="Link to this heading">#</a></h2> 583<p>Letâs define helper functions to plot the graphs.</p> 584<div class="cell docutils container">
585<div class="cell_input docutils container"> 586<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">create_plots</span><span class="p">():</span> 587 <span class="n">plt</span><span class="o">.</span><span class="n">ioff</span><span class="p">()</span> 588 <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">16</span> <span class="o">/</span> <span class="mf">9.0</span> <span class="o">*</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4</span> <span class="o">*</span> <span class="mi">1</span><span class="p">),</span> <span class="n">layout</span><span class="o">=</span><span class="s2">"constrained"</span><span class="p">)</span> 589 <span class="n">fig</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">"Gradient Descent"</span><span class="p">)</span> 590 <span class="n">ax0</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> 591 <span class="n">ax0</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Spending"</span><span class="p">,</span> <span class="n">fontweight</span><span class="o">=</span><span class="s2">"normal"</span><span class="p">)</span> 592 <span class="n">ax0</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Sales"</span><span class="p">,</span> <span class="n">fontweight</span><span class="o">=</span><span class="s2">"normal"</span><span class="p">)</span> 593 <span class="n">ax0</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Linear Regression"</span><span class="p">)</span> 594 595 <span class="n">ax1</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">projection</span><span class="o">=</span><span class="s2">"3d"</span><span class="p">)</span> 596 <span class="n">ax1</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Slope, w"</span><span class="p">)</span> 597 <span class="n">ax1</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Intercept, b"</span><span class="p">)</span> 598 <span class="n">ax1</span><span class="o">.</span><span class="n">set_zlabel</span><span class="p">(</span><span class="s2">
598"Error"</span><span class="p">)</span> 599 <span class="n">ax1</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Error"</span><span class="p">)</span> 600 <span class="n">ax1</span><span class="o">.</span><span class="n">view_init</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="o">-</span><span class="mi">35</span><span class="p">)</span> 601 <span class="n">ax1</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span> 602 603 <span class="n">camera</span> <span class="o">=</span> <span class="n">Camera</span><span class="p">(</span><span class="n">fig</span><span class="p">)</span> 604 <span class="k">return</span> <span class="n">ax0</span><span class="p">,</span> <span class="n">ax1</span><span class="p">,</span> <span class="n">camera</span> 605 606 607<span class="k">def</span><span class="w"> </span><span class="nf">generate_error_range</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">N</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">b_max</span><span class="p">):</span> 608 <span class="n">w_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">N</span><span class="p">)</span> 609 <span class="n">b_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">b_max</span><span class="p">,</span> <span class="n">N</span><span class="p">)</span> 610 <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">meshgrid</span><span class="p">(</span><span class="n">w_vals</span><span class="p">,</span> <span class="n">b_vals</span><span class="p">)</span> 611 612 <span class="n">error_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">w_range</span><span class="p">)</span> 613 <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">N</span><span class="p">):</span> 614 <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">N</span><span class="p">):</span> 615 <span class="n">error_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">],</span> <span class="n">b_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">])</span> 616 617 <span class="k">return</span> <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span> 618</pre></div> 619</div> 620</div> 621</div> 622</section> 623<section id="training-the-model"> 624<h2>Training the model<a class="headerlink" href="#training-the-model" title="Link to this heading">#</a></h2> 625<p>The train function will update the parameters in each epoch and update the visualization.</p> 626<div class="cell docutils container"> 627<div class="cell_input docutils container"> 628<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w0</span><span class="p">,</span> <span class="n">b0</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">,</span> <span class="n">epochs</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">):</span> 629 <span class="n">w</span> <span class="o">=</span> <span class="n">w0</span> 630 <span class="n">b</span> <span class="o">=</span> <span class="n">b0</span> 631 632 <span class="n">ax0</span><span class="p">,</span> <span class="n">ax1</span><span class="p">,</span> <span class="n">camera</span> <span class="o">=</span> <span class="n">create_plots</span><span class="p">()</span> 633 <span class="n">loss_dims</span> <span class="o">=</span> <span class="mi">20</span> 634 <span class="n">w_max</span> <span class="o">=</span> <span class="mf">0.5</span> 635 <span class="n">b_max</span> <span class="o">=</span> <span class="mi">15</span> 636 <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span> <span class="o">=</span> <span class="n">generate_error_range</span><span class="p">(</span>
637 <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">loss_dims</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">b_max</span> 638 <span class="p">)</span> 639 640 <span class="n">X_plot</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span> 641 642 <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span> 643 <span class="c1"># Capture and log at specific milestones and the final epoch</span> 644 <span class="k">if</span> <span class="p">(</span> 645 <span class="p">(</span><span class="n">e</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span> 646 <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o"><</span> <span class="mi">60</span> <span class="ow">and</span> <span class="n">e</span> <span class="o">%</span> <span class="mi">5</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span> 647 <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o">%</span> <span class="mi">1000</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span> 648 <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o">==</span> <span class="n">epochs</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> 649 <span class="p">):</span> 650 <span class="c1"># Redraw the loss landscape each frame so it appears in animation output.</span> 651 <span class="n">ax1</span><span class="o">.</span><span class="n">plot_wireframe</span><span class="p">(</span> 652 <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mf">0.7</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.9</span> 653 <span class="p">)</span> 654 <span class="n">ax1</span><span class="o">.</span><span class="n">scatter</span><span class="p">([</span><span class="n">w</span><span class="p">],</span> <span class="p">[</span><span class="n">b</span><span class="p">],</span> <span class="p">[</span><span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)],</span> <span class="n">color</span><span class="o">=</span><span class="s2">"red"</span><span class="p">,</span> <span class="n">s</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span> 655 656 <span class="c1"># Plot regression data and current prediction line</span> 657 <span class="n">ax0</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"o"</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span> 658 <span class="n">ax0</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">X_plot</span><span class="p">,</span> <span class="n">X_plot</span> <span class="o">*</span> <span class="n">w</span> <span class="o">+</span> <span class="n">b</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"black"</span><span class="p">)</span> 659
660 <span class="c1"># Log current training progress</span> 661 <span class="n">current_loss</span> <span class="o">=</span> <span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> 662 <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"epoch: </span><span class="si">{</span><span class="n">e</span><span class="si">:</span><span class="s2">4d</span><span class="si">}</span><span class="s2"> | loss: </span><span class="si">{</span><span class="n">current_loss</span><span class="si">:</span><span class="s2">.8f</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span> 663 <span class="n">camera</span><span class="o">.</span><span class="n">snap</span><span class="p">()</span> 664 665 <span class="c1"># Update parameters simultaneously</span> 666 <span class="n">w</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">update_w_and_b</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">)</span> 667 668 <span class="n">animation</span> <span class="o">=</span> <span class="n">camera</span><span class="o">.</span><span class="n">animate</span><span class="p">()</span> 669 <span class="n">animation</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">output_filename</span><span class="p">,</span> <span class="n">writer</span><span class="o">=</span><span class="s2">"pillow"</span><span class="p">)</span> 670 <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span> 671 672 <span class="k">return</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span> 673</pre></div> 674</div> 675</div> 676</div> 677<p>Letâs train the linear regression model on a sample dataset.</p> 678<div class="cell docutils container"> 679<div class="cell_input docutils container"> 680<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fname</span> <span class="o">=</span> <span class="s2">"REGRESSION-gradientDescent-data.txt"</span> 681<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">loadtxt</span><span class="p">(</span><span class="n">fname</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s2">","</span><span class="p">,</span> <span class="n">unpack</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">skiprows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">usecols</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span> 682<span class="n">output_filename</span> <span class="o">=</span> <span class="s2">"gradient_descent.gif"</span> 683<span class="n">train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mf">0.00005</span><span class="p">,</span> <span class="mi">4000</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">)</span> 684</pre></div> 685</div> 686</div> 687<div class="cell_output docutils container"> 688<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>epoch: 0 | loss: 223.71625000 689epoch: 5 | loss: 124.87289357 690epoch: 10 | loss: 80.08948497 691epoch: 15 | loss: 59.79711689 692epoch: 20 | loss: 50.60004436 693epoch: 25 | loss: 46.42952831 694epoch: 30 | loss: 44.53621857 695epoch: 35 | loss: 43.67456373 696epoch: 40 | loss: 43.28028563 697epoch: 45 | loss: 43.09774660 698epoch: 50 | loss: 43.01113387 699epoch: 55 | loss: 42.96798022 700epoch: 1000 | loss: 41.62139410 701epoch: 2000 | loss: 40.30390096 702epoch: 3000 | loss: 39.06018012 703epoch: 3999 | loss: 37.88724135 704</pre></div> 705</div> 706<img alt="../_images/d232dade55d20b5d93ecff63ed6b7fdf346ea415b62b25dd3479798079efe00f.png" src="../_images/d232dade55d20b5d93ecff63ed6b7fdf346ea415b62b25dd3479798079efe00f.png" /> 707<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(np.float64(0.4560126699891239), np.float64(1.026884217380755)) 708</pre></div> 709</div> 710</div> 711</div> 712<div class="cell docutils container">
713<div class="cell_input docutils container"> 714<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">Image</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">output_filename</span><span class="p">)</span> 715</pre></div> 716</div> 717</div> 718<div class="cell_output docutils container"> 719<img alt="../_images/a66db7b49a7fe07413cf8ba5683575a1b6382a0673d4c9cfd89cd08cd5582885.gif" src="../_images/a66db7b49a7fe07413cf8ba5683575a1b6382a0673d4c9cfd89cd08cd5582885.gif" /> 720</div> 721</div> 722</section> 723</section> 724 725
725<script type="text/x-thebe-config"> 726 { 727 requestKernel: true, 728 binderOptions: { 729 repo: "binder-examples/jupyter-stacks-datascience", 730 ref: "master", 731 }, 732 codeMirrorConfig: { 733 theme: "abcdef", 734 mode: "python" 735 }, 736 kernelOptions: { 737 name: "python3", 738 path: "./chapter1" 739 }, 740 predefinedOutput: true 741 } 742 </script>
742 743
743<script>kernelName = 'python3'</script>
743 744 745 </article> 746 747 748 749 750 751 752 <footer class="prev-next-footer"> 753 754<div class="prev-next-area"> 755 <a class="left-prev" 756 href="../index.html" 757 title="previous page"> 758 <i class="fa-solid fa-angle-left"></i> 759 <div class="prev-next-info"> 760 <p class="prev-next-subtitle">previous</p> 761 <p class="prev-next-title">Machine Learning Visualized</p> 762 </div> 763 </a> 764 <a class="right-next" 765 href="optimizers.html" 766 title="next page"> 767 <div class="prev-next-info"> 768 <p class="prev-next-subtitle">next</p> 769 <p class="prev-next-title">Optimizers</p> 770 </div> 771 <i class="fa-solid fa-angle-right"></i> 772 </a> 773</div> 774 </footer> 775 776 </div> 777 778 779 780 <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner"> 781 782 783 <div class="sidebar-secondary-item"> 784 <div class="page-toc tocsection onthispage"> 785 <i class="fa-solid fa-list"></i> Contents 786 </div> 787 <nav class="bd-toc-nav page-toc"> 788 <ul class="visible nav section-nav flex-column"> 789<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-dataset">Training Dataset</a></li> 790<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-mean-squared-error">Loss Function: Mean Squared Error</a></li> 791<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-gradient">Loss Function Gradient</a></li> 792<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-w">Loss Function in Terms of W</a></li> 793<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-b">Loss Function in Terms of b</a></li> 794<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-linear-regression-model">Training the Linear Regression Model</a></li> 795<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#graphing-functions">Graphing functions</a></li> 796<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-model">Training the model</a></li> 797</ul> 798 </nav></div> 799 800</div></div> 801 802 803 </div> 804 <footer class="bd-footer-content"> 805 806<div class="bd-footer-content__inner container"> 807 808 <div class="footer-item"> 809 810<p class="component-author"> 811By Gavin Hung 812</p> 813 814 </div> 815 816 <div class="footer-item"> 817 818 819 </div> 820 821 <div class="footer-item"> 822 823 </div> 824 825 <div class="footer-item"> 826 827 </div> 828 829</div> 830 </footer> 831 832 833 </main> 834 </div> 835 </div> 836 837 <!-- Scripts loaded after <body> so the DOM is not blocked --> 838
838<script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script>
vendor: 1 bytes, line 838
838
839<script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script>
839 840 841 <footer class="bd-footer"> 842 </footer> 843 </body> 844</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.