PageSourceSearch

https://ml-visualized.com/chapter1/linear_regression.html

html ml-visualized.com collected 2026-10-03 19:59:15 UTC 53,592 bytes, 844 lines download raw bytes

1
2<!DOCTYPE html>
3
4
5<html lang="en" data-content_root="../" data-theme="light">
6
7  <head>
8    <meta charset="utf-8" />
9    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
10
11    <title>Gradient Descent &#8212; Machine Learning Visualized</title>
12  
13  
14  
15  
15<script data-cfasync="false">
16    document.documentElement.dataset.mode = localStorage.getItem("mode") || "light";
17    document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
18  </script>
18
19  
20  <!-- Loaded before other Sphinx assets -->
21  <link href="../_static/styles/theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
22<link href="../_static/styles/bootstrap.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
23<link href="../_static/styles/pydata-sphinx-theme.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
24
25  
26  <link href="../_static/vendor/fontawesome/6.5.1/css/all.min.css?digest=8d27b9dea8ad943066ae" rel="stylesheet" />
27  <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-solid-900.woff2" />
28<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-brands-400.woff2" />
29<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.1/webfonts/fa-regular-400.woff2" />
30
31    <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=03e43079" />
32    <link rel="stylesheet" type="text/css" href="../_static/styles/sphinx-book-theme.css?v=a3416100" />
33    <link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=9c3e77be" />
34    <link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
35    <link rel="stylesheet" type="text/css" href="../_static/mystnb.11b39860a7a0cbfd473a3ad8a317855267ff0bd372690045ca344a6b62be495e.css" />
36    <link rel="stylesheet" type="text/css" href="../_static/sphinx-thebe.css?v=4fa983c6" />
37    <link rel="stylesheet" type="text/css" href="../_static/sphinx-design.min.css?v=95c83b7e" />
38    <link rel="stylesheet" type="text/css" href="../_static/custom.css?v=3a394b7d" />
39  
40  <!-- Pre-loaded scripts that we'll load fully later -->
41  <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae" />
42<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae" />
43  
43<script src="../_static/vendor/fontawesome/6.5.1/js/all.min.js?digest=8d27b9dea8ad943066ae"></script>
43
44
45    
45<script src="../_static/documentation_options.js?v=9eb32ce0"></script>
45
46    
46<script src="../_static/doctools.js?v=9a2dae69"></script>
46
47    
47<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
47
48    
48<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
48
49    
49<script src="../_static/copybutton.js?v=f281be69"></script>
49
50    
50<script src="../_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
50
51    
51<script>let toggleHintShow = 'Click to show';</script>
51
52    
52<script>let toggleHintHide = 'Click to hide';</script>
52
53    
53<script>let toggleOpenOnPrint = 'true';</script>
53
54    
54<script src="../_static/togglebutton.js?v=b8b12719"></script>
54
55    
55<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
55
56    
56<script src="../_static/design-tabs.js?v=f930bc37"></script>
vendor: 76 bytes, lines 56-57
56
57    <script async="async" src="https://www.googletagmanager.com/gtag/js?id=
57G-9S2CQ3B60T
vendor: 16 bytes, lines 57-58
57"></script>
58    
58<script>
59                
vendor: 177 bytes, lines 59-62
59window.dataLayer = window.dataLayer || [];
60                function gtag(){ dataLayer.push(arguments); }
61                gtag('js', new Date());
62                gtag('config', '
62G-9S2CQ3B60T
vendor: 16 bytes, lines 62-63
62');
63            
63</script>
63
64    
64<script>const THEBE_JS_URL = "https://unpkg.com/[email protected]/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
64
65    
65<script async="async" src="../_static/sphinx-thebe.js?v=c100c467"></script>
65
66    
66<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
66
67    
67<script>
68                
vendor: 177 bytes, lines 68-71
68window.dataLayer = window.dataLayer || [];
69                function gtag(){ dataLayer.push(arguments); }
70                gtag('js', new Date());
71                gtag('config', '
71G-9S2CQ3B60T
vendor: 16 bytes, lines 71-72
71');
72            
72</script>
72
73    
73<script>const THEBE_JS_URL = "https://unpkg.com/[email protected]/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
73
74    
74<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
74
75    
75<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
75
76    
76<script>DOCUMENTATION_OPTIONS.pagename = 'chapter1/linear_regression';</script>
76
77    <link rel="icon" href="../_static/logo.png"/>
78    <link rel="index" title="Index" href="../genindex.html" />
79    <link rel="search" title="Search" href="../search.html" />
80    <link rel="next" title="Optimizers" href="optimizers.html" />
81    <link rel="prev" title="Machine Learning Visualized" href="../index.html" />
82  <meta name="viewport" content="width=device-width, initial-scale=1"/>
83  <meta name="docsearch:language" content="en"/>
84  </head>
85  
86  
87  <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="light">
88
89  
90  
91  <a id="pst-skip-link" class="skip-link" href="#main-content">Skip to main content</a>
92  
93  <div id="pst-scroll-pixel-helper"></div>
94  
95  <button type="button" class="btn rounded-pill" id="pst-back-to-top">
96    <i class="fa-solid fa-arrow-up"></i>
97    Back to top
98  </button>
99
100  
101  <input type="checkbox"
102          class="sidebar-toggle"
103          name="__primary"
104          id="__primary"/>
105  <label class="overlay overlay-primary" for="__primary"></label>
106  
107  <input type="checkbox"
108          class="sidebar-toggle"
109          name="__secondary"
110          id="__secondary"/>
111  <label class="overlay overlay-secondary" for="__secondary"></label>
112  
113  <div class="search-button__wrapper">
114    <div class="search-button__overlay"></div>
115    <div class="search-button__search-container">
116<form class="bd-search d-flex align-items-center"
117      action="../search.html"
118      method="get">
119  <i class="fa-solid fa-magnifying-glass"></i>
120  <input type="search"
121         class="form-control"
122         name="q"
123         id="search-input"
124         placeholder="Search this book..."
125         aria-label="Search this book..."
126         autocomplete="off"
127         autocorrect="off"
128         autocapitalize="off"
129         spellcheck="false"/>
130  <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
131</form></div>
132  </div>
133  
134    <header class="bd-header navbar navbar-expand-lg bd-navbar">
135    </header>
136  
137
138  <div class="bd-container">
139    <div class="bd-container__inner bd-page-width">
140      
141      
142      
143      <div class="bd-sidebar-primary bd-sidebar">
144        
145
146  
147  <div class="sidebar-header-items sidebar-primary__section">
148    
149    
150    
151    
152  </div>
153  
154    <div class="sidebar-primary-items__start sidebar-primary__section">
155        <div class="sidebar-primary-item">
156
157  
158
159<a class="navbar-brand logo" href="../index.html">
160  
161  
162  
163  
164  
165  
166    <p class="title logo__title">Machine Learning Visualized</p>
167  
168</a></div>
169        <div class="sidebar-primary-item">
170
171 
171<script>
172 document.write(`
173   <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
174    <i class="fa-solid fa-magnifying-glass"></i>
175    <span class="search-button__default-text">Search</span>
176    <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
177   </button>
178 `);
179 </script>
179</div>
180        <div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
181    <div class="bd-toc-item navbar-nav active">
182        
183        <ul class="nav bd-sidenav bd-sidenav__home-link">
184            <li class="toctree-l1">
185                <a class="reference internal" href="../index.html">
186                    Machine Learning Visualized
187                </a>
188            </li>
189        </ul>
190        <p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 1. Optimization</span></p>
191<ul class="current nav bd-sidenav">
192<li class="toctree-l1 current active"><a class="current reference internal" href="#">Gradient Descent</a></li>
193<li class="toctree-l1"><a class="reference internal" href="optimizers.html">Optimizers</a></li>
194<li class="toctree-l1"><a class="reference internal" href="interactive_linear_regression.html">Interactive Linear Regression</a></li>
195</ul>
196<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 2. Clustering and Reduction</span></p>
197<ul class="nav bd-sidenav">
198<li class="toctree-l1"><a class="reference internal" href="../chapter2/pca.html">Principal Component Analysis</a></li>
199<li class="toctree-l1"><a class="reference internal" href="../chapter2/k_means.html">K-Means Clustering</a></li>
200</ul>
201<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 3. Linear Models</span></p>
202<ul class="nav bd-sidenav">
203<li class="toctree-l1"><a class="reference internal" href="../chapter3/perceptron.html">Perceptron</a></li>
204<li class="toctree-l1"><a class="reference internal" href="../chapter3/logistic_regression.html">Logistic Regression</a></li>
205<li class="toctree-l1"><a class="reference internal" href="../chapter3/interactive_perceptron.html">Interactive Perceptron</a></li>
206<li class="toctree-l1"><a class="reference internal" href="../chapter3/interactive_logistic_regression.html">Interactive Logistic Regression</a></li>
207</ul>
208<p aria-level="2" class="caption" role="heading"><span class="caption-text">Chapter 4. Neural Networks</span></p>
209<ul class="nav bd-sidenav">
210<li class="toctree-l1"><a class="reference internal" href="../chapter4/neural_network.html">Neural Network</a></li>
211<li class="toctree-l1"><a class="reference internal" href="../chapter4/neural_network_weights.html">Neural Network Weights Visualization</a></li>
212<li class="toctree-l1"><a class="reference internal" href="../chapter4/autoencoder.html">Autoencoder</a></li>
213</ul>
214
215    </div>
216</nav></div>
217    </div>
218  
219  
220  <div class="sidebar-primary-items__end sidebar-primary__section">
221  </div>
222  
223  <div id="rtd-footer-container"></div>
224
225
226      </div>
227      
228      <main id="main-content" class="bd-main">
229        
230        
231
232<div class="sbt-scroll-pixel-helper"></div>
233
234          <div class="bd-content">
235            <div class="bd-article-container">
236              
237              <div class="bd-header-article">
238<div class="header-article-items header-article__inner">
239  
240    <div class="header-article-items__start">
241      
242        <div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
243  <span class="fa-solid fa-bars"></span>
244</button></div>
245      
246    </div>
247  
248  
249    <div class="header-article-items__end">
250      
251        <div class="header-article-item">
252
253<div class="article-header-buttons">
254
255
256
257
258
259<div class="dropdown dropdown-source-buttons">
260  <button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Source repositories">
261    <i class="fab fa-github"></i>
262  </button>
263  <ul class="dropdown-menu">
264      
265      
266      
267      <li><a href="https://github.com/gavinkhung/machine-learning-visualized" target="_blank"
268   class="btn btn-sm btn-source-repository-button dropdown-item"
269   title="Source repository"
270   data-bs-placement="left" data-bs-toggle="tooltip"
271>
272  
273
274<span class="btn__icon-container">
275  <i class="fab fa-github"></i>
276  </span>
277<span class="btn__text-container">Repository</span>
278</a>
279</li>
280      
281      
282      
283      
284      <li>
284<a href="https://github.com/gavinkhung/machine-learning-visualized/issues/new?title=Issue%20on%20page%20%2Fchapter1/linear_regression.html&body=Your%20issue%20content%20here." target="_blank"
285   class="btn btn-sm btn-source-issues-button dropdown-item"
286   title="Open an issue"
287   data-bs-placement="left" data-bs-toggle="tooltip"
288>
289  
290
291<span class="btn__icon-container">
292  <i class="fas fa-lightbulb"></i>
293  </span>
294<span class="btn__text-container">Open issue</span>
295</a>
296</li>
297      
298  </ul>
299</div>
300
301
302
303
304
305
306<div class="dropdown dropdown-download-buttons">
307  <button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
308    <i class="fas fa-download"></i>
309  </button>
310  <ul class="dropdown-menu">
311      
312      
313      
314      <li><a href="../_sources/chapter1/linear_regression.ipynb" target="_blank"
315   class="btn btn-sm btn-download-source-button dropdown-item"
316   title="Download source file"
317   data-bs-placement="left" data-bs-toggle="tooltip"
318>
319  
320
321<span class="btn__icon-container">
322  <i class="fas fa-file"></i>
323  </span>
324<span class="btn__text-container">.ipynb</span>
325</a>
326</li>
327      
328      
329      
330      
331      <li>
332<button onclick="window.print()"
333  class="btn btn-sm btn-download-pdf-button dropdown-item"
334  title="Print to PDF"
335  data-bs-placement="left" data-bs-toggle="tooltip"
336>
337  
338
339<span class="btn__icon-container">
340  <i class="fas fa-file-pdf"></i>
341  </span>
342<span class="btn__text-container">.pdf</span>
343</button>
344</li>
345      
346  </ul>
347</div>
348
349
350
351
352<button onclick="toggleFullScreen()"
353  class="btn btn-sm btn-fullscreen-button"
354  title="Fullscreen mode"
355  data-bs-placement="bottom" data-bs-toggle="tooltip"
356>
357  
358
359<span class="btn__icon-container">
360  <i class="fas fa-expand"></i>
361  </span>
362
363</button>
364
365
366
367<script>
368document.write(`
369  <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
370    <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
371    <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
372    <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
373  </button>
374`);
375</script>
375
376
377
378<script>
379document.write(`
380  <button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
381    <i class="fa-solid fa-magnifying-glass fa-lg"></i>
382  </button>
383`);
384</script>
384
385<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
386    <span class="fa-solid fa-list"></span>
387</button>
388</div></div>
389      
390    </div>
391  
392</div>
393</div>
394              
395              
396
397<div id="jb-print-docs-body" class="onlyprint">
398    <h1>Gradient Descent</h1>
399    <!-- Table of contents -->
400    <div id="print-main-content">
401        <div id="jb-print-toc">
402            
403            <div>
404                <h2> Contents </h2>
405            </div>
406            <nav aria-label="Page">
407                <ul class="visible nav section-nav flex-column">
408<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-dataset">Training Dataset</a></li>
409<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-mean-squared-error">Loss Function: Mean Squared Error</a></li>
410<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-gradient">Loss Function Gradient</a></li>
411<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-w">Loss Function in Terms of W</a></li>
412<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-b">Loss Function in Terms of b</a></li>
413<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-linear-regression-model">Training the Linear Regression Model</a></li>
414<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#graphing-functions">Graphing functions</a></li>
415<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-model">Training the model</a></li>
416</ul>
417            </nav>
418        </div>
419    </div>
420</div>
421
422              
423                
424<div id="searchbox"></div>
425                <article class="bd-article">
426                  
427  <section class="tex2jax_ignore mathjax_ignore" id="gradient-descent">
428<h1>Gradient Descent<a class="headerlink" href="#gradient-descent" title="Link to this heading">#</a></h1>
429<p>Gradient Descent is an algorithm that finds the local minimum of a function. This is applicable to machine learning because we want to find the optimal parameters that minimize our loss function. In machine learning, loss functions quantify the amount of error between the predicted values from a machine learning model and the actual expected values. In this notebook, we will perform linear regression by using gradient descent to find the optimal slope and y-intercept.</p>
430<section id="training-dataset">
431<h2>Training Dataset<a class="headerlink" href="#training-dataset" title="Link to this heading">#</a></h2>
432<p>Importing the libraries</p>
433<div class="cell docutils container">
434<div class="cell_input docutils container">
435<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">np</span>
436<span class="kn">import</span><span class="w"> </span><span class="nn">matplotlib.pyplot</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">plt</span>
437<span class="kn">import</span><span class="w"> </span><span class="nn">scienceplots</span>
438<span class="kn">from</span><span class="w"> </span><span class="nn">IPython.display</span><span class="w"> </span><span class="kn">import</span> <span class="n">display</span><span class="p">,</span> <span class="n">Latex</span><span class="p">,</span> <span class="n">Image</span>
439<span class="kn">from</span><span class="w"> </span><span class="nn">celluloid</span><span class="w"> </span><span class="kn">import</span> <span class="n">Camera</span>
440
441<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
442<span class="n">plt</span><span class="o">.</span><span class="n">style</span><span class="o">.</span><span class="n">
442use</span><span class="p">([</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"no-latex"</span><span class="p">])</span>
443</pre></div>
444</div>
445</div>
446</div>
447<p>Let’s look at the training dataset. We will use columns 2 and 4 of the txt file. The linear regression model will find the optimal slope and y-intercept to fit the data.</p>
448<div class="cell docutils container">
449<div class="cell_input docutils container">
450<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fname</span> <span class="o">=</span> <span class="s2">"REGRESSION-gradientDescent-data.txt"</span>
451<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">loadtxt</span><span class="p">(</span><span class="n">fname</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s2">","</span><span class="p">,</span> <span class="n">unpack</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">skiprows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">usecols</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span>
452
453<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
454<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">()</span>
455<span class="n">ax</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"o"</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span>
456</pre></div>
457</div>
458</div>
459<div class="cell_output docutils container">
460<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;matplotlib.collections.PathCollection at 0x7a9104ff5e20&gt;
461</pre></div>
462</div>
463<img alt="../_images/fa4746a5ae55598f4a4e28ae287ca7cc834b20a984c460cef12a8165c03f109e.png" src="../_images/fa4746a5ae55598f4a4e28ae287ca7cc834b20a984c460cef12a8165c03f109e.png" />
464</div>
465</div>
466</section>
467<section id="loss-function-mean-squared-error">
468<h2>Loss Function: Mean Squared Error<a class="headerlink" href="#loss-function-mean-squared-error" title="Link to this heading">#</a></h2>
469<p>For the linear regression model, the predicted value <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the product of the weight <span class="math notranslate nohighlight">\(w\)</span> and the input feature <span class="math notranslate nohighlight">\(x_i\)</span> plus a bias term <span class="math notranslate nohighlight">\(b\)</span>:</p>
470<div class="math notranslate nohighlight">
471\[\hat{y}_i = w x_i + b\]</div>
472<p>We will use the mean squared error function as our loss function:</p>
473<div class="math notranslate nohighlight">
474\[\begin{split}
475\begin{align*}
476MSE &amp;= \frac{1}{n} \sum_{i=1}^{n}(y_{i}-\hat{y}_i)^2 \\
477&amp;= \frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2
478\end{align*}
479\end{split}\]</div>
480</section>
481<section id="loss-function-gradient">
482<h2>Loss Function Gradient<a class="headerlink" href="#loss-function-gradient" title="Link to this heading">#</a></h2>
483<div class="cell docutils container">
484<div class="cell_input docutils container">
485<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
486    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">)))</span>
487</pre></div>
488</div>
489</div>
490</div>
491<p>In each epoch of gradient descent, a parameter is updated by subtracting the product of the gradient of the function and the learning rate (<span class="math notranslate nohighlight">\(\eta\)</span>). The learning rate controls how much the parameters should change. Small learning rates are precise, but are slow. Large learning rates are fast, but may overshoot and prevent the model from finding the minimum.</p>
492<div class="math notranslate nohighlight">
493\[
494\begin{align*}
495X_{t+1} = X_t - \eta \nabla f(X_t)
496\end{align*}
497\]</div>
498<p>Since we are finding the optimal slope (<span class="math notranslate nohighlight">\(w\)</span>) and y-intercept (<span class="math notranslate nohighlight">\(b\)</span>) for our linear regression model, we must find the partial derivatives of the loss function with respect to <span class="math notranslate nohighlight">\(w\)</span> and <span class="math notranslate nohighlight">\(b\)</span>.</p>
499</section>
500<section id="loss-function-in-terms-of-w">
501<h2>Loss Function in Terms of W<a class="headerlink" href="#loss-function-in-terms-of-w" title="Link to this heading">#</a></h2>
502<p>Loss function with respect to <span class="math notranslate nohighlight">\(w\)</span>:</p>
503<div class="math notranslate nohighlight">
504\[\begin{split}
505\begin{align*}
506\frac{\partial}{\partial w} \left( MSE \right) &amp;= \frac{\partial}{\partial w}\left[\frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2\right] \\
507&amp;= \frac{1}{n} \sum_{i=1}^{n} \frac{\partial}{\partial w}\left[(y_{i}-(w x_{i} + b))^2\right] \\
508&amp;= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\frac{\partial}{\partial w}\left[y_{i}-(w x_{i} + b)\right] \\
509&amp;= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))(-x_{i}) \\
510&amp;= -\frac{2}{n} \sum_{i=1}^{n} x_{i}(y_{i}-(w x_{i} + b))
511\end{align*}
512\end{split}\]</div>
513<div class="cell docutils container">
514<div class="cell_input docutils container">
515<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss_dw</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
516    <span class="k">return</span> <span class="o">-</span><span class="mi">2</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span> <span class="o">*</span> <span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">)))</span>
517</pre></div>
518</div>
519</div>
520</div>
521</section>
522<section id="loss-function-in-terms-of-b">
523<h2>Loss Function in Terms of b<a class="headerlink" href="#loss-function-in-terms-of-b" title="Link to this heading">#</a></h2>
524<p>Loss function with respect to <span class="math notranslate nohighlight">\(b\)</span>:</p>
525<div class="math notranslate nohighlight">
526\[\begin{split}
527\begin{align*}
528\frac{\partial}{\partial b} \left( MSE \right) &amp;= \frac{\partial}{\partial b}\left[\frac{1}{n} \sum_{i=1}^{n}(y_{i}-(w x_{i} + b))^2\right] \\
529&amp;= \frac{1}{n} \sum_{i=1}^{n} \frac{\partial}{\partial b}\left[(y_{i}-(w x_{i} + b))^2\right] \\
530&amp;= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\frac{\partial}{\partial b}\left[y_{i}-(w x_{i} + b)\right] \\
531&amp;= \frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))(-1) \\
532&amp;= -\frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))
533\end{align*}
534\end{split}\]</div>
535<div class="cell docutils container">
536<div class="cell_input docutils container">
537<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">mse_loss_db</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
538    <span class="k">return</span> <span class="o">-</span><span class="mi">2</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span> <span class="o">-</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="p">))</span>
539</pre></div>
540</div>
541</div>
542</div>
543</section>
544<section id="training-the-linear-regression-model">
545<h2>Training the Linear Regression Model<a class="headerlink" href="#training-the-linear-regression-model" title="Link to this heading">#</a></h2>
546<p>Let’s define a function that uses the gradient algorithm to update the parameters of the loss function. The function uses the gradient functions we derived earlier.</p>
547<p>General Gradient Descent Equation:</p>
548<div class="math notranslate nohighlight">
549\[X_{t+1} = X_t - \eta \nabla f(X_t)\]</div>
550<p>Weights Gradient Descent:</p>
551<div class="math notranslate nohighlight">
552\[\begin{split}
553\begin{align*}
554w &amp;= w - \eta \frac{\partial}{\partial w} [L(w, b)] \\
555&amp;= w - \eta \left[-\frac{2}{n} \sum_{i=1}^{n} x_{i}(y_{i}-(w x_{i} + b))\right]
556\end{align*}
557\end{split}\]</div>
558<p>Bias Gradient Descent:</p>
559<div class="math notranslate nohighlight">
560\[\begin{split}
561\begin{align*}
562b &amp;= b - \eta \frac{\partial}{\partial b} [L(w, b)] \\
563&amp;= b - \eta \left[-\frac{2}{n} \sum_{i=1}^{n} (y_{i}-(w x_{i} + b))\right]
564\end{align*}
565\end{split}\]</div>
566<div class="cell docutils container">
567<div class="cell_input docutils container">
568<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">update_w_and_b</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">):</span>
569    <span class="n">dw</span> <span class="o">=</span> <span class="n">mse_loss_dw</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
570    <span class="n">db</span> <span class="o">=</span> <span class="n">mse_loss_db</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
571
572    <span class="n">w</span> <span class="o">=</span> <span class="n">w</span> <span class="o">-</span> <span class="n">dw</span> <span class="o">*</span> <span class="n">learning_rate</span>
573    <span class="n">b</span> <span class="o">=</span> <span class="n">b</span> <span class="o">-</span> <span class="n">db</span> <span class="o">*</span> <span class="n">learning_rate</span>
574
575    <span class="k">return</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span>
576</pre></div>
577</div>
578</div>
579</div>
580</section>
581<section id="graphing-functions">
582<h2>Graphing functions<a class="headerlink" href="#graphing-functions" title="Link to this heading">#</a></h2>
583<p>Let’s define helper functions to plot the graphs.</p>
584<div class="cell docutils container">
585<div class="cell_input docutils container">
586<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">create_plots</span><span class="p">():</span>
587    <span class="n">plt</span><span class="o">.</span><span class="n">ioff</span><span class="p">()</span>
588    <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">16</span> <span class="o">/</span> <span class="mf">9.0</span> <span class="o">*</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4</span> <span class="o">*</span> <span class="mi">1</span><span class="p">),</span> <span class="n">layout</span><span class="o">=</span><span class="s2">"constrained"</span><span class="p">)</span>
589    <span class="n">fig</span><span class="o">.</span><span class="n">suptitle</span><span class="p">(</span><span class="s2">"Gradient Descent"</span><span class="p">)</span>
590    <span class="n">ax0</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
591    <span class="n">ax0</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Spending"</span><span class="p">,</span> <span class="n">fontweight</span><span class="o">=</span><span class="s2">"normal"</span><span class="p">)</span>
592    <span class="n">ax0</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Sales"</span><span class="p">,</span> <span class="n">fontweight</span><span class="o">=</span><span class="s2">"normal"</span><span class="p">)</span>
593    <span class="n">ax0</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Linear Regression"</span><span class="p">)</span>
594
595    <span class="n">ax1</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">projection</span><span class="o">=</span><span class="s2">"3d"</span><span class="p">)</span>
596    <span class="n">ax1</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Slope, w"</span><span class="p">)</span>
597    <span class="n">ax1</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Intercept, b"</span><span class="p">)</span>
598    <span class="n">ax1</span><span class="o">.</span><span class="n">set_zlabel</span><span class="p">(</span><span class="s2">
598"Error"</span><span class="p">)</span>
599    <span class="n">ax1</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Error"</span><span class="p">)</span>
600    <span class="n">ax1</span><span class="o">.</span><span class="n">view_init</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="o">-</span><span class="mi">35</span><span class="p">)</span>
601    <span class="n">ax1</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>
602
603    <span class="n">camera</span> <span class="o">=</span> <span class="n">Camera</span><span class="p">(</span><span class="n">fig</span><span class="p">)</span>
604    <span class="k">return</span> <span class="n">ax0</span><span class="p">,</span> <span class="n">ax1</span><span class="p">,</span> <span class="n">camera</span>
605
606
607<span class="k">def</span><span class="w"> </span><span class="nf">generate_error_range</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">N</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">b_max</span><span class="p">):</span>
608    <span class="n">w_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">N</span><span class="p">)</span>
609    <span class="n">b_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">b_max</span><span class="p">,</span> <span class="n">N</span><span class="p">)</span>
610    <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">meshgrid</span><span class="p">(</span><span class="n">w_vals</span><span class="p">,</span> <span class="n">b_vals</span><span class="p">)</span>
611
612    <span class="n">error_range</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="n">w_range</span><span class="p">)</span>
613    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">N</span><span class="p">):</span>
614        <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">N</span><span class="p">):</span>
615            <span class="n">error_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">],</span> <span class="n">b_range</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">])</span>
616
617    <span class="k">return</span> <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span>
618</pre></div>
619</div>
620</div>
621</div>
622</section>
623<section id="training-the-model">
624<h2>Training the model<a class="headerlink" href="#training-the-model" title="Link to this heading">#</a></h2>
625<p>The train function will update the parameters in each epoch and update the visualization.</p>
626<div class="cell docutils container">
627<div class="cell_input docutils container">
628<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w0</span><span class="p">,</span> <span class="n">b0</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">,</span> <span class="n">epochs</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">):</span>
629    <span class="n">w</span> <span class="o">=</span> <span class="n">w0</span>
630    <span class="n">b</span> <span class="o">=</span> <span class="n">b0</span>
631
632    <span class="n">ax0</span><span class="p">,</span> <span class="n">ax1</span><span class="p">,</span> <span class="n">camera</span> <span class="o">=</span> <span class="n">create_plots</span><span class="p">()</span>
633    <span class="n">loss_dims</span> <span class="o">=</span> <span class="mi">20</span>
634    <span class="n">w_max</span> <span class="o">=</span> <span class="mf">0.5</span>
635    <span class="n">b_max</span> <span class="o">=</span> <span class="mi">15</span>
636    <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span> <span class="o">=</span> <span class="n">generate_error_range</span><span class="p">(</span>
637        <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">loss_dims</span><span class="p">,</span> <span class="n">w_max</span><span class="p">,</span> <span class="n">b_max</span>
638    <span class="p">)</span>
639
640    <span class="n">X_plot</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
641
642    <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span>
643        <span class="c1"># Capture and log at specific milestones and the final epoch</span>
644        <span class="k">if</span> <span class="p">(</span>
645            <span class="p">(</span><span class="n">e</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span>
646            <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o">&lt;</span> <span class="mi">60</span> <span class="ow">and</span> <span class="n">e</span> <span class="o">%</span> <span class="mi">5</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span>
647            <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o">%</span> <span class="mi">1000</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span>
648            <span class="ow">or</span> <span class="p">(</span><span class="n">e</span> <span class="o">==</span> <span class="n">epochs</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
649        <span class="p">):</span>
650            <span class="c1"># Redraw the loss landscape each frame so it appears in animation output.</span>
651            <span class="n">ax1</span><span class="o">.</span><span class="n">plot_wireframe</span><span class="p">(</span>
652                <span class="n">w_range</span><span class="p">,</span> <span class="n">b_range</span><span class="p">,</span> <span class="n">error_range</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mf">0.7</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.9</span>
653            <span class="p">)</span>
654            <span class="n">ax1</span><span class="o">.</span><span class="n">scatter</span><span class="p">([</span><span class="n">w</span><span class="p">],</span> <span class="p">[</span><span class="n">b</span><span class="p">],</span> <span class="p">[</span><span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)],</span> <span class="n">color</span><span class="o">=</span><span class="s2">"red"</span><span class="p">,</span> <span class="n">s</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
655
656            <span class="c1"># Plot regression data and current prediction line</span>
657            <span class="n">ax0</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"#1f77b4"</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s2">"o"</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span>
658            <span class="n">ax0</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">X_plot</span><span class="p">,</span> <span class="n">X_plot</span> <span class="o">*</span> <span class="n">w</span> <span class="o">+</span> <span class="n">b</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s2">"black"</span><span class="p">)</span>
659
660            <span class="c1"># Log current training progress</span>
661            <span class="n">current_loss</span> <span class="o">=</span> <span class="n">mse_loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span>
662            <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"epoch: </span><span class="si">{</span><span class="n">e</span><span class="si">:</span><span class="s2">4d</span><span class="si">}</span><span class="s2"> | loss: </span><span class="si">{</span><span class="n">current_loss</span><span class="si">:</span><span class="s2">.8f</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
663            <span class="n">camera</span><span class="o">.</span><span class="n">snap</span><span class="p">()</span>
664
665        <span class="c1"># Update parameters simultaneously</span>
666        <span class="n">w</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">update_w_and_b</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">learning_rate</span><span class="p">)</span>
667
668    <span class="n">animation</span> <span class="o">=</span> <span class="n">camera</span><span class="o">.</span><span class="n">animate</span><span class="p">()</span>
669    <span class="n">animation</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">output_filename</span><span class="p">,</span> <span class="n">writer</span><span class="o">=</span><span class="s2">"pillow"</span><span class="p">)</span>
670    <span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
671
672    <span class="k">return</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span>
673</pre></div>
674</div>
675</div>
676</div>
677<p>Let’s train the linear regression model on a sample dataset.</p>
678<div class="cell docutils container">
679<div class="cell_input docutils container">
680<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fname</span> <span class="o">=</span> <span class="s2">"REGRESSION-gradientDescent-data.txt"</span>
681<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">loadtxt</span><span class="p">(</span><span class="n">fname</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s2">","</span><span class="p">,</span> <span class="n">unpack</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">skiprows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">usecols</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span>
682<span class="n">output_filename</span> <span class="o">=</span> <span class="s2">"gradient_descent.gif"</span>
683<span class="n">train</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mf">0.00005</span><span class="p">,</span> <span class="mi">4000</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">)</span>
684</pre></div>
685</div>
686</div>
687<div class="cell_output docutils container">
688<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>epoch:    0 | loss: 223.71625000
689epoch:    5 | loss: 124.87289357
690epoch:   10 | loss: 80.08948497
691epoch:   15 | loss: 59.79711689
692epoch:   20 | loss: 50.60004436
693epoch:   25 | loss: 46.42952831
694epoch:   30 | loss: 44.53621857
695epoch:   35 | loss: 43.67456373
696epoch:   40 | loss: 43.28028563
697epoch:   45 | loss: 43.09774660
698epoch:   50 | loss: 43.01113387
699epoch:   55 | loss: 42.96798022
700epoch: 1000 | loss: 41.62139410
701epoch: 2000 | loss: 40.30390096
702epoch: 3000 | loss: 39.06018012
703epoch: 3999 | loss: 37.88724135
704</pre></div>
705</div>
706<img alt="../_images/d232dade55d20b5d93ecff63ed6b7fdf346ea415b62b25dd3479798079efe00f.png" src="../_images/d232dade55d20b5d93ecff63ed6b7fdf346ea415b62b25dd3479798079efe00f.png" />
707<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(np.float64(0.4560126699891239), np.float64(1.026884217380755))
708</pre></div>
709</div>
710</div>
711</div>
712<div class="cell docutils container">
713<div class="cell_input docutils container">
714<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">Image</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">output_filename</span><span class="p">)</span>
715</pre></div>
716</div>
717</div>
718<div class="cell_output docutils container">
719<img alt="../_images/a66db7b49a7fe07413cf8ba5683575a1b6382a0673d4c9cfd89cd08cd5582885.gif" src="../_images/a66db7b49a7fe07413cf8ba5683575a1b6382a0673d4c9cfd89cd08cd5582885.gif" />
720</div>
721</div>
722</section>
723</section>
724
725    
725<script type="text/x-thebe-config">
726    {
727        requestKernel: true,
728        binderOptions: {
729            repo: "binder-examples/jupyter-stacks-datascience",
730            ref: "master",
731        },
732        codeMirrorConfig: {
733            theme: "abcdef",
734            mode: "python"
735        },
736        kernelOptions: {
737            name: "python3",
738            path: "./chapter1"
739        },
740        predefinedOutput: true
741    }
742    </script>
742
743    
743<script>kernelName = 'python3'</script>
743
744
745                </article>
746              
747
748              
749              
750              
751              
752                <footer class="prev-next-footer">
753                  
754<div class="prev-next-area">
755    <a class="left-prev"
756       href="../index.html"
757       title="previous page">
758      <i class="fa-solid fa-angle-left"></i>
759      <div class="prev-next-info">
760        <p class="prev-next-subtitle">previous</p>
761        <p class="prev-next-title">Machine Learning Visualized</p>
762      </div>
763    </a>
764    <a class="right-next"
765       href="optimizers.html"
766       title="next page">
767      <div class="prev-next-info">
768        <p class="prev-next-subtitle">next</p>
769        <p class="prev-next-title">Optimizers</p>
770      </div>
771      <i class="fa-solid fa-angle-right"></i>
772    </a>
773</div>
774                </footer>
775              
776            </div>
777            
778            
779              
780                <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
781
782
783  <div class="sidebar-secondary-item">
784  <div class="page-toc tocsection onthispage">
785    <i class="fa-solid fa-list"></i> Contents
786  </div>
787  <nav class="bd-toc-nav page-toc">
788    <ul class="visible nav section-nav flex-column">
789<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-dataset">Training Dataset</a></li>
790<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-mean-squared-error">Loss Function: Mean Squared Error</a></li>
791<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-gradient">Loss Function Gradient</a></li>
792<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-w">Loss Function in Terms of W</a></li>
793<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loss-function-in-terms-of-b">Loss Function in Terms of b</a></li>
794<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-linear-regression-model">Training the Linear Regression Model</a></li>
795<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#graphing-functions">Graphing functions</a></li>
796<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#training-the-model">Training the model</a></li>
797</ul>
798  </nav></div>
799
800</div></div>
801              
802            
803          </div>
804          <footer class="bd-footer-content">
805            
806<div class="bd-footer-content__inner container">
807  
808  <div class="footer-item">
809    
810<p class="component-author">
811By Gavin Hung
812</p>
813
814  </div>
815  
816  <div class="footer-item">
817    
818
819  </div>
820  
821  <div class="footer-item">
822    
823  </div>
824  
825  <div class="footer-item">
826    
827  </div>
828  
829</div>
830          </footer>
831        
832
833      </main>
834    </div>
835  </div>
836  
837  <!-- Scripts loaded after <body> so the DOM is not blocked -->
838  
838<script src="../_static/scripts/bootstrap.js?digest=8d27b9dea8ad943066ae"></script>
vendor: 1 bytes, line 838
838
839<script src="../_static/scripts/pydata-sphinx-theme.js?digest=8d27b9dea8ad943066ae"></script>
839
840
841  <footer class="bd-footer">
842  </footer>
843  </body>
844</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.