PageSourceSearch

https://smartcorelib.github.io/user_guide/supervised.html

html smartcorelib.github.io collected 2026-10-03 09:57:53 UTC 110,918 bytes, 935 lines download raw bytes

1<!DOCTYPE html>
2<html lang="en-US">
3  <head>
4    
5        <!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 64 bytes, lines 5-6
5
6<script async src="https://www.googletagmanager.com/gtag/js?id=
6G-D1L0JVQB04
vendor: 12 bytes, line 6
6"></script>
7<script>
8  
vendor: 134 bytes, lines 8-12
8window.dataLayer = window.dataLayer || [];
9  function gtag(){dataLayer.push(arguments);}
10  gtag('js', new Date());
11
12  gtag('config', '
12G-D1L0JVQB04
vendor: 4 bytes, line 12
12');
13</script>
13
14
15    
16    <meta charset="utf-8">
17    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
18    <title>Supervised Learning - SmartCore</title>
19
20    <link rel="apple-touch-icon-precomposed" sizes="57x57" href="/assets/favicon/apple-touch-icon-57x57.png" />
21<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/assets/favicon/apple-touch-icon-114x114.png" />
22<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/assets/favicon/apple-touch-icon-72x72.png" />
23<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/assets/favicon/apple-touch-icon-144x144.png" />
24<link rel="apple-touch-icon-precomposed" sizes="60x60" href="/assets/favicon/apple-touch-icon-60x60.png" />
25<link rel="apple-touch-icon-precomposed" sizes="120x120" href="/assets/favicon/apple-touch-icon-120x120.png" />
26<link rel="apple-touch-icon-precomposed" sizes="76x76" href="/assets/favicon/apple-touch-icon-76x76.png" />
27<link rel="apple-touch-icon-precomposed" sizes="152x152" href="/assets/favicon/apple-touch-icon-152x152.png" />
28<link rel="icon" type="image/png" href="/assets/favicon/favicon-196x196.png" sizes="196x196" />
29<link rel="icon" type="image/png" href="/assets/favicon/favicon-96x96.png" sizes="96x96" />
30<link rel="icon" type="image/png" href="/assets/favicon/favicon-32x32.png" sizes="32x32" />
31<link rel="icon" type="image/png" href="/assets/favicon/favicon-16x16.png" sizes="16x16" />
32<link rel="icon" type="image/png" href="/assets/favicon/favicon-128.png" sizes="128x128" />
33<meta name="application-name" content="&nbsp;"/>
34<meta name="msapplication-TileColor" content="#FFFFFF" />
35<meta name="msapplication-TileImage" content="/assets/favicon/mstile-144x144.png" />
36<meta name="msapplication-square70x70logo" content="/assets/favicon/mstile-70x70.png" />
37<meta name="msapplication-square150x150logo" content="/assets/favicon/mstile-150x150.png" />
38<meta name="msapplication-wide310x150logo" content="/assets/favicon/mstile-310x150.png" />
39<meta name="msapplication-square310x310logo" content="/assets/favicon/mstile-310x310.png" />
40
41
42    <link rel="stylesheet" href="/assets/css/main.css">
43    <link rel="stylesheet" href="https://cdn.rawgit.com/afeld/bootstrap-toc/v1.0.1/dist/bootstrap-toc.min.css"/>
44    <link rel="shortcut icon" type="image/png" href="/favicon.png">  
45    <!-- Begin Jekyll SEO tag v2.8.0 -->
46<title>Supervised Learning | SmartCore</title>
47<meta name="generator" content="Jekyll v3.10.0" />
48<meta property="og:title" content="Supervised Learning" />
49<meta name="author" content="SmartCore Developers" />
50<meta property="og:locale" content="en_US" />
51<meta name="description" content="Supervised learning with Smartcore, including, but not limited to KNN, Naive Bayes, Decision Trees, ensemble methods, Random Forest, linear models and SVM." />
52<meta property="og:description" content="Supervised learning with Smartcore, including, but not limited to KNN, Naive Bayes, Decision Trees, ensemble methods, Random Forest, linear models and SVM." />
53<link rel="canonical" href="https://smartcorelib.github.io/user_guide/supervised.html" />
54<meta property="og:url" content="https://smartcorelib.github.io/user_guide/supervised.html" />
55<meta property="og:site_name" content="SmartCore" />
56<meta property="og:type" content="website" />
57<meta name="twitter:card" content="summary" />
58<meta property="twitter:title" content="Supervised Learning" />
59<script type="application/ld+json">
60{"@context":"https://schema.org","@type":"WebPage","author":{"@type":"Person","name":"SmartCore Developers"},"description":"Supervised learning with Smartcore, including, but not limited to KNN, Naive Bayes, Decision Trees, ensemble methods, Random Forest, linear models and SVM.","headline":"Supervised Learning","url":"https://smartcorelib.github.io/user_guide/supervised.html"}</script>
60
61<!-- End Jekyll SEO tag -->
62
63  </head>
64  <body>
65    <nav class="navbar navbar-expand-md navbar-dark bg-dark">
66  <div class="container">
67  <a class="navbar-brand text-primary font-weight-bold mt-1" href="/">
68    <img src="/assets/logo/smartcore.png" width="50" height="50" class="d-inline-block align-middle" alt="">
69    SmartCore
70  </a>
71  <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNav">
72      <span class="navbar-toggler-icon"></span>
73  </button>
74  <div class="collapse navbar-collapse" id="navbarNav">          
75      <ul class="navbar-nav mr-auto">        
76        
77          <li class="nav-item ">            
78            
79              <a class="nav-link font-weight-bold align-middle" href="/user_guide/quick_start.html">Quick Start</a>
80              
81          </li> 
82        
83          <li class="nav-item ">            
84                      
85              <a class="nav-link font-weight-bold align-middle" href="https://docs.rs/smartcore/0.6/smartcore/">API</a>
86              
87          </li> 
88        
89          <li class="nav-item ">            
90                      
91              <a class="nav-link font-weight-bold align-middle" href="https://github.com/smartcorelib/smartcore-jupyter">Examples</a>
92              
93          </li> 
94        
95          <li class="nav-item ">            
96            
97              <a class="nav-link font-weight-bold align-middle" href="/user_guide/benchmarks.html">Benchmarks</a>
98              
99          </li> 
100        
101        <li class="nav-item dropdown">
102          <a class="nav-link dropdown-toggle font-weight-bold align-middle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
103            User Guide
104          </a>
105          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
106            
107              
108                <a class="dropdown-item" href="/user_guide/quick_start.html">Getting Started</a>
109              
110            
111              
112                <a class="dropdown-item" href="/user_guide/supervised.html">Supervised Learning</a>
113              
114            
115              
116                <a class="dropdown-item" href="/user_guide/unsupervised.html">Unsupervised Learning</a>
117              
118            
119              
120                <a class="dropdown-item" href="/user_guide/preprocessing.html">Preprocessing</a>
121              
122            
123              
124                <a class="dropdown-item" href="/user_guide/model_selection.html">Model Selection</a>
125              
126            
127              
128                <a class="dropdown-item" href="/user_guide/benchmarks.html">Benchmarks</a>
129              
130            
131              
132                <a class="dropdown-item" href="/user_guide/developer.html">Developer's Guide</a>
133              
134            
135          </div>
136        </li>           
137      </ul>
138      <ul class="navbar-nav ml-auto">                     
139        
140          <li class="nav-item ">
141            
142              <a class="nav-link font-weight-bold align-middle" href="/about.html">About Us</a>
143            
144          </li> 
145        
146        <li class="nav-item">
147          <a class="nav-link" href="https://github.com/smartcorelab/smartcore">
148            <span><i class="fab fa-github" aria-hidden="true"></i></span>
149          </a>
150        </li>
151      </ul>
152  </div>
153  </div>
154</nav>
155
156    <div class="container">
157    
158        <div class="row flex-xl-nowrap">            
159        
160            <div class="col-md-3">
161                <div class=" bg-light rounded mt-3 p-3 text-center font-weight-bold text-muted">
162                    SmartCore 0.6
163                    <br/>
164                    <a class="align-bottom font-weight-light" href="/user_guide/versions.html">other version</a>
165                </div>                
166                <nav id="toc" class="sticky-top bg-light rounded mt-3 p-3"></nav>
167            </div>
168            
169            <div manual="true" class="col-md-9 mt-3 text-justify" markdown="1">                            
170
171                <h1 id="supervised-learning">Supervised Learning</h1>
172
173<p>Most machine learning problems falls into one of two categories: <em>supervised</em> and <em>unsupervised</em>.
174This page describes supervised learning algorithms implemented in <em>SmartCore</em>.</p>
175
176<p>In supervised learning we build a model which can be written in the very general form as
177\[Y = f(X) + \epsilon\]
178\(X = (X_1,X_2,…,X_p)\) are observations that consist of \(p\) different predictors, \(Y\) is an associated target values and 
179\(\epsilon\) is a random error term, which is independent of \(X\) and has zero mean.
180We fit an unknown function \(f\) to our data to predict the response for future observations or better understand the relationship between the response and the predictors.</p>
181
182<p>Supervised learning is by far the most common machine learning method that is used in practice. Supervised learning problems can be grouped into regression and classification:</p>
183
184<ul>
185  <li><em>Classification</em>: when the output variable is qualitative (category), such as “cancerous” or “benign”, “round” or “square”.</li>
186  <li><em>Regression</em>: a regression problem is when the output variable is quantitative, such as “height” or “dollars”.</li>
187</ul>
188
189<p>All algorithms in <em>SmartCore</em>
189 support both, qualitative and quantitative response variables and follow the same naming convention for function names. 
190To fit an algorithm to your data use <code class="language-plaintext highlighter-rouge">fit</code> method that takes 2 mandatory parameters: <code class="language-plaintext highlighter-rouge">x</code> for your predictors and <code class="language-plaintext highlighter-rouge">y</code> for target values. All optional parameters are hidden behind <code class="language-plaintext highlighter-rouge">Default::default()</code>.
191To make a prediction use <code class="language-plaintext highlighter-rouge">predict</code> method that takes new observations as <code class="language-plaintext highlighter-rouge">x</code> and predicts estimated class labels or target values.</p>
192
193<h2 id="k-nearest-neighbors">K Nearest Neighbors</h2>
194
195<p>K-nearest neighbors (KNN) is one of the simplest and best-known non-parametric classification and regression method.
196KNN does not require training. The algorithm simply stores the entire dataset and then uses this dataset to make predictions.</p>
197
198<p>More formally,
199given a positive integer \(K\) and a test observation \(x_0\), the KNN classifier first identifies the \(K\) points in the training data that are closest to \(x_0\), represented by \(N_0\) and then estimates the conditional probability for class \(j\) as the fraction of points in N0 whose response values equal \(j\):</p>
200
201<p>\[ Pr(Y=j \vert X=x_0) = \frac{1}{K} \sum_{i \in N_0} I(y_i=j) \]</p>
202
203<p>KNN Regressor is closely related to the KNN classifier. It estimates target value using the average of all the reponses in \(N_0\), i.e.</p>
204
205<p>\[ \hat{y} = \frac{1}{K} \sum_{i \in N_0} y_i \]</p>
206
207<p>The choice of \(K\) is very important. \(K\) can be found by tuning algorithm on a holdout dataset. It is a good idea to try many different values for \(K\) (e.g. values from 1 to 21) and see which value gives the best test error rate.</p>
208
209<p>To determine which of the \(K\) instances in the training dataset are most similar to a new input a <a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/index.html">distance metric</a> is used. 
210For real-valued input variables, the most popular distance metric is <a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/euclidian/index.html">Euclidean distance</a>. You can choose the best distance metric based on the properties of your data. If you are unsure, you can experiment with different distance metrics and different values of \(K\) together and see which mix results in the most accurate models.</p>
211
212<h3 id="nearest-neighbors-classification">Nearest Neighbors Classification</h3>
213
214<p>To fit KNN Classifier to your data use <a href="https://docs.rs/smartcore/0.6/smartcore/neighbors/knn_classifier/struct.KNNClassifier.html"><code class="language-plaintext highlighter-rouge">KNNClassifier</code></a>. Let’s fit KNN Classifier to the <a href="https://docs.rs/smartcore/0.6/smartcore/dataset/breast_cancer/index.html">Breast Cancer</a> dataset:</p>
215
216<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
217<span class="c1">// DenseMatrix wrapper around Vec</span>
218<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
219<span class="c1">// Imports for KNN classifier</span>
220<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">neighbors</span><span class="p">::</span><span class="nn">knn_classifier</span><span class="p">::</span><span class="n">KNNClassifier</span><span class="p">;</span>
221<span class="c1">// Model performance</span>
222<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">accuracy</span><span class="p">;</span>
223<span class="k">
223use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
224<span class="c1">// Load dataset</span>
225<span class="k">let</span> <span class="n">cancer_data</span> <span class="o">=</span> <span class="nn">breast_cancer</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
226<span class="c1">// Transform dataset into a NxM matrix</span>
227<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">cancer_data</span><span class="py">.num_samples</span><span class="p">)</span>
228    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">cancer_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
229    <span class="nf">.collect</span><span class="p">();</span>
230<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
231<span class="c1">// These are our target class labels</span>
232<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">cancer_data</span><span class="py">.target</span><span class="p">;</span>
233<span class="c1">// Split dataset into training/test (80%/20%)</span>
234<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
235<span class="c1">// KNN classifier</span>
236<span class="k">let</span> <span class="n">y_hat_knn</span> <span class="o">=</span> <span class="nn">KNNClassifier</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span>
237    <span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span>
238    <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>        
239    <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">(),</span>
240<span class="p">)</span><span class="nf">.and_then</span><span class="p">(|</span><span class="n">knn</span><span class="p">|</span> <span class="n">knn</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>    
241<span class="c1">// Calculate test error</span>
242<span class="nd">println!</span><span class="p">(</span><span class="s">"accuracy: {}"</span><span class="p">,</span> <span class="nf">accuracy</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_knn</span><span class="p">));</span>
243</code></pre></div></div>
244
245<p>Default value of \(K\) is 3. If you want to change value of this and other parameters replace <code class="language-plaintext highlighter-rouge">Default::default()</code> with an instance of <a href="https://docs.rs/smartcore/0.6/smartcore/neighbors/knn_classifier/struct.KNNClassifierParameters.html"><code class="language-plaintext highlighter-rouge">KNNClassifierParameters</code></a>.</p>
246
247<h3 id="nearest-neighbors-regression">Nearest Neighbors Regression</h3>
248
249<p>KNN Regressor, implemented in <a href="https://docs.rs/smartcore/0.6/smartcore/neighbors/knn_regressor/struct.KNNRegressor.html"><code class="language-plaintext highlighter-rouge">KNNClassifier</code></a> is very similar to KNN Classifier, the only difference is that returned value is a real value instead of class label. To fit <code class="language-plaintext highlighter-rouge">KNNRegressor</code> to <a href="https://docs.rs/smartcore/0.6/smartcore/dataset/boston/index.html">Boston Housing</a> dataset:</p>
250
251<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">
251use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
252<span class="c1">// DenseMatrix wrapper around Vec</span>
253<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
254<span class="c1">// KNN</span>
255<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="nn">distance</span><span class="p">::</span><span class="n">Distances</span><span class="p">;</span>
256<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">neighbors</span><span class="p">::</span><span class="nn">knn_regressor</span><span class="p">::{</span><span class="n">KNNRegressor</span><span class="p">,</span> <span class="n">KNNRegressorParameters</span><span class="p">};</span>
257<span class="c1">// Model performance</span>
258<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
259<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
260<span class="c1">// Load dataset</span>
261<span class="k">let</span> <span class="n">boston_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
262<span class="c1">// Transform dataset into a NxM matrix</span>
263<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">boston_data</span><span class="py">.num_samples</span><span class="p">)</span>
264    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">boston_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
265    <span class="nf">.collect</span><span class="p">();</span>
266<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
267<span class="c1">// These are our target class labels</span>
268<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">boston_data</span><span class="py">.target</span><span class="p">;</span>
269<span class="c1">// Split dataset into training/test (80%/20%)</span>
270<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
270Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
271<span class="c1">// KNN regressor</span>
272<span class="k">let</span> <span class="n">y_hat_knn</span> <span class="o">=</span> <span class="nn">KNNRegressor</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span>
273    <span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span>
274    <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>    
275    <span class="nn">KNNRegressorParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span><span class="nf">.with_distance</span><span class="p">(</span><span class="nn">Distances</span><span class="p">::</span><span class="nf">euclidian</span><span class="p">()),</span>
276<span class="p">)</span><span class="nf">.and_then</span><span class="p">(|</span><span class="n">knn</span><span class="p">|</span> <span class="n">knn</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>
277<span class="c1">// Calculate test error</span>
278<span class="nd">println!</span><span class="p">(</span><span class="s">"MSE: {}"</span><span class="p">,</span> <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_knn</span><span class="p">));</span>
279</code></pre></div></div>
280
281<p>As with KNN Classifier you can change value of k and other parameters by passing an instance of <a href="https://docs.rs/smartcore/0.6/smartcore/neighbors/knn_regressor/struct.KNNRegressorParameters.html"><code class="language-plaintext highlighter-rouge">KNNRegressorParameters</code></a> to <code class="language-plaintext highlighter-rouge">fit</code> function.</p>
282
283<h3 id="nearest-neighbor-algorithms">Nearest Neighbor Algorithms</h3>
284
285<p>The computational complexity of KNN increases with the size of the training dataset. This is because every time prediction is made algorithm has to search through all stored samples to find K nearest neighbors. Efficient implementation of KNN requires special data structure, like <a href="https://en.wikipedia.org/wiki/Cover_tree">CoverTree</a> to speed up look-up of nearest neighbors during prediction.</p>
286
287<p>Cover Tree is the default algorithm for KNN regressor and classifier. Change value of <code class="language-plaintext highlighter-rouge">algorithm</code> field of the <code class="language-plaintext highlighter-rouge">KNNRegressorParameters</code> or <code class="language-plaintext highlighter-rouge">KNNClassifierParameters</code> if you want to switch to brute force search method.</p>
288
289<h4 id="brute-force">Brute Force</h4>
290
291<p>The brute force nearest neighbor search is the simplest algorithm that calculates the distance from the query point to every other point in the dataset while maintaining a list of K nearest items in a <a href="https://en.wikipedia.org/wiki/Binary_heap#Search">Binary Heap</a>. This algorithms does not maintain any search data structure and results in \(O(n)\) search time, where \(n\) is number of samples. Brute force search algorithm is implemented in <a href="https://docs.rs/smartcore/0.6/smartcore/algorithm/neighbour/linear_search/index.html">LinearKNNSearch</a>.</p>
292
293<h4 id="cover-tree">Cover Tree</h4>
294
295<p>Although Brute Force algorithms is very simple approach it outperforms a lot of space partitioning approaches like <a href="https://en.wikipedia.org/wiki/K-d_tree">k-d tree</a> on higher dimensional spaces. However, the brute-force approach quickly becomes infeasible as the dataset grows in size. To address inefficiencies of Brute Force other data structures are used that reduce the required number of distance calculations by efficiently encoding aggregate distance information for the sample.</p>
296
297<p>A <a href="https://docs.rs/smartcore/0.6/smartcore/algorithm/neighbour/cover_tree/index.html">Cover Tree</a> is a tree data structure used for the partitiong of metric spaces to speed up nearest neighbor operations. Cover trees are fast in practice and have great theoretical properties:</p>
298
299<ul>
300  <li>Construction: \(O(c^6n\log n)\)</li>
301  <li>Query:  \(O(c^{12}\log n)\),</li>
302  <li>The cover tree requires  \(O(n)\) space.</li>
303</ul>
304
305<p>
305where \(n\) is number of samples in a dataset and \(c\) denotes the expansion constant.</p>
306
307<h3 id="distance-metrics">Distance Metrics</h3>
308
309<p>The choice of distance metric for KNN algorithm largely depends on properties of your data. If you don’t know which distance to use go with Euclidean distance function or choose metric that gives you the best performance on a hold out test set. 
310There are many other distance measures that can be used with KNN in <em>SmartCore</em></p>
311
312<table class="table table-striped table-bordered">
313  <thead>
314    <tr>
315      <th style="text-align: center">Distance Metric</th>
316      <th style="text-align: center">Parameters</th>
317      <th style="text-align: center">Description</th>
318    </tr>
319  </thead>
320  <tbody>
321    <tr>
322      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/euclidian/index.html">Euclidian</a></td>
323      <td style="text-align: center"> </td>
324      <td style="text-align: center">\(\sqrt{\sum_{i=1}^n (x_i-y_i)^2}\)</td>
325    </tr>
326    <tr>
327      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/hamming/index.html">Hamming</a></td>
328      <td style="text-align: center"> </td>
329      <td style="text-align: center">the number of places where \( x \) and \( y \) differ</td>
330    </tr>
331    <tr>
332      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/mahalanobis/index.html">Mahalanobis</a></td>
333      <td style="text-align: center">\(S\), covariance matrix of the dataset</td>
334      <td style="text-align: center">\(\sqrt{(x - y)^TS^{-1}(x - y)}\)</td>
335    </tr>
336    <tr>
337      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/manhattan/index.html">Manhattan</a></td>
338      <td style="text-align: center"> </td>
339      <td style="text-align: center">\(\sum_{i=0}^n \lvert x_i - y_i \rvert\)</td>
340    </tr>
341    <tr>
342      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/minkowski/index.html">Minkowski</a></td>
343      <td style="text-align: center">p, distance order</td>
344      <td style="text-align: center">\(\left(\sum_{i=0}^n \lvert x_i - y_i \rvert^p\right)^{1/p}\)</td>
345    </tr>
346    <tr>
347      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/cosine/index.html">Cosine</a></td>
348      <td style="text-align: center"> </td>
349      <td style="text-align: center">\(1 - \frac{x \cdot y}{\lVert x \rVert \lVert y \rVert}\)</td>
350    </tr>
351    <tr>
352      <td style="text-align: center"><a href="https://docs.rs/smartcore/0.6/smartcore/metrics/distance/jaccard/index.html">Jaccard</a></td>
353      <td style="text-align: center"> </td>
354      <td style="text-align: center">\(\frac{\lvert x \cup y \rvert - \lvert x \cap y \rvert}{\lvert x \cup y \rvert}\)</td>
355    </tr>
356  </tbody>
357</table>
358
359<h2 id="linear-models">Linear Models</h2>
360
361<p>Linear regression is one of the most well known and well understood algorithms in statistics and machine learning.</p>
362
363<figure class="image" align="center">
364  <img src="/assets/imgs/simple_regression.svg" alt="Simple linear regression" class="img-fluid" />
365  <figcaption>Figure 1. Simple linear regression.</figcaption>
366</figure>
367
368<p>The model describes the relationship between a dependent variable y (also called the response) as a function of one or more independent, or explanatory variables \(X_i\). The general equation for a linear model is:
369\[y = \beta_0 + \sum_{i=1}^n \beta_iX_i + \epsilon\]</p>
370
371<p>where the target value \(y\) is a linear combination of the features \(X_i\).</p>
372
373<h3 id="linear-regression">Linear Regression</h3>
374
375<p>Use <code class="language-plaintext highlighter-rouge">fit</code> method of <a href="https://docs.rs/smartcore/0.6/smartcore/linear/linear_regression/index.html"><code class="language-plaintext highlighter-rouge">LinearRegression</code></a> to fit Ordinary Least Squares to your data.</p>
376
377<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">
377use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
378<span class="c1">// DenseMatrix wrapper around Vec</span>
379<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
380<span class="c1">// Linear Regression</span>
381<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linear</span><span class="p">::</span><span class="nn">linear_regression</span><span class="p">::</span><span class="n">LinearRegression</span><span class="p">;</span>
382<span class="c1">// Model performance</span>
383<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
384<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
385<span class="c1">// Load dataset</span>
386<span class="k">let</span> <span class="n">cancer_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
387<span class="c1">// Transform dataset into a NxM matrix</span>
388<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">cancer_data</span><span class="py">.num_samples</span><span class="p">)</span>
389    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">cancer_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
390    <span class="nf">.collect</span><span class="p">();</span>
391<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
392<span class="c1">// These are our target class labels</span>
393<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">cancer_data</span><span class="py">.target</span><span class="p">;</span>
394<span class="c1">// Split dataset into training/test (80%/20%)</span>
395<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
396<span class="c1">// Linear Regression</span>
397<span class="k">let</span> <span class="n">y_hat_lr</span> <span class="o">=</span> <span class="nn">LinearRegression</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span> <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">())</span>
398    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">lr</span><span class="p">|</span> <span class="n">lr</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>
399<span class="c1">// Calculate test error</span>
400<span class="nd">println!</span><span class="p">(</span><span class="s">"MSE: {}"</span><span class="p">,</span> <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_lr</span><span class="p">));</span>
401</code></pre></div></div>
402
403<p>By default, <em>SmartCore</em> uses <a href="https://en.wikipedia.org/wiki/Singular_value_decomposition">SVD Decomposition</a> to find estimates of \(\beta_i\) that minimizes the sum of the squared residuals. While SVD Decomposition provides the most stable solution, you might decide to go with <a href="https://en.wikipedia.org/wiki/QR_decomposition">QR Decomposition</a>
403 since this approach is more computationally efficient than SVD Decomposition. For comparison, runtime complexity of SVD Decomposition is \(O(mn^2 + n^3)\) vs \(O(mn^2 + n^3/3)\) for QR decomposition, where \(n\) and \(m\) are dimentions of input matrix \(X\). Use <code class="language-plaintext highlighter-rouge">solver</code> attribute of the <a href="https://docs.rs/smartcore/0.6/smartcore/linear/linear_regression/struct.LinearRegressionParameters.html"><code class="language-plaintext highlighter-rouge">LinearRegressionParameters</code></a> to choose between decomposition methods.</p>
404
405<h3 id="shrinkage-methods">Shrinkage Methods</h3>
406
407<p>One way to avoid overfitting when you fit a linear model to your dataset is to use regularization. In simple terms, regularization shrinks parameters of the model towards zero. This shrinkage has the effect of reducing variance. Depending on what type of shrinkage is performed, some of the coefficients may be estimated to be exactly zero. Hence, shrinkage methods can also perform variable selection.</p>
408
409<h4 id="ridge-regression">Ridge Regression</h4>
410
411<p>Ridge Regression is a regularized version of linear regression that adds L2 regularization term to the cost function:</p>
412
413<p>\[\lambda \sum_{i=i}^n \beta_i^2\]</p>
414
415<p>where \(\lambda \geq 0\) is a tuning hyperparameter. If \(\lambda\) is close to 0, then it has no effects because Ridge Regression is similar to plain linear regression. As \(\lambda\) gets larger the shrinking effect on the weights gets stronger and the weights approach zero.</p>
416
417<p>To fit Ridge Regression use structs from the <a href="https://docs.rs/smartcore/0.6/smartcore/linear/ridge_regression/index.html"><code class="language-plaintext highlighter-rouge">ridge_regression</code></a> module:</p>
418
419<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
420<span class="c1">// DenseMatrix wrapper around Vec</span>
421<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
422<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linear</span><span class="p">::</span><span class="nn">ridge_regression</span><span class="p">::{</span><span class="n">RidgeRegression</span><span class="p">,</span> <span class="n">RidgeRegressionParameters</span><span class="p">};</span>
423<span class="c1">// Model performance</span>
424<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
425<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
426<span class="c1">// Load dataset</span>
427<span class="k">let</span> <span class="n">boston_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
428<span class="c1">// Transform dataset into a NxM matrix</span>
429<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">boston_data</span><span class="py">.num_samples</span><span class="p">)</span>
430    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">boston_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
431    <span class="nf">.collect</span><span class="p">();</span>
432<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
433<span class="c1">// These are our target values</span>
434<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">boston_data</span><span class="py">.target</span><span class="p">;</span>
435<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
435Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
436<span class="c1">// Ridge Regression</span>
437<span class="k">let</span> <span class="n">y_hat_rr</span> <span class="o">=</span> <span class="nn">RidgeRegression</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span>
438    <span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span>
439    <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>
440    <span class="nn">RidgeRegressionParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span><span class="nf">.with_alpha</span><span class="p">(</span><span class="mf">0.5</span><span class="p">),</span>
441<span class="p">)</span>
442<span class="nf">.and_then</span><span class="p">(|</span><span class="n">rr</span><span class="p">|</span> <span class="n">rr</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span>
443<span class="nf">.unwrap</span><span class="p">();</span>
444<span class="c1">// Calculate test error</span>
445<span class="nd">println!</span><span class="p">(</span>
446    <span class="s">"MSE Ridge Regression: {}"</span><span class="p">,</span>
447    <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_rr</span><span class="p">)</span>
448<span class="p">);</span>
449</code></pre></div></div>
450
451<h4 id="lasso">LASSO</h4>
452
453<p>LASSO stands for Least Absolute Shrinkage and Selection Operator. It is analogous to Ridge Regression but uses L1 regularization term instead of L2 regularization term:</p>
454
455<p>\[\lambda \sum_{i=i}^n \mid \beta_i \mid \]</p>
456
457<p>As with ridge regression, the lasso shrinks the coefficient estimates towards zero. However, in the case of the lasso, the L1 penalty has the effect of forcing some of the coefficient estimates to be exactly equal to zero when the tuning parameter \(\lambda\) is sufficiently large. Hence, the lasso performs variable selection and models generated from the lasso are generally much easier to interpret than those produced by ridge regression.</p>
458
459<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
460<span class="c1">// DenseMatrix wrapper around Vec</span>
461<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
462<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linear</span><span class="p">::</span><span class="nn">lasso</span><span class="p">::{</span><span class="n">Lasso</span><span class="p">,</span> <span class="n">LassoParameters</span><span class="p">};</span>
463<span class="c1">// Model performance</span>
464<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
465<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
466<span class="c1">// Load dataset</span>
467<span class="k">let</span> <span class="n">boston_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
468<span class="c1">// Transform dataset into a NxM matrix</span>
469<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">boston_data</span><span class="py">.num_samples</span><span class="p">)</span>
470    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">boston_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
471    <span class="nf">.collect</span><span class="p">();</span>
472<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
473<span class="c1">// These are our target values</span>
474<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">boston_data</span><span class="py">.target</span><span class="p">;</span>
475<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
475Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
476<span class="c1">// LASSO</span>
477<span class="k">let</span> <span class="n">y_hat_lasso</span> <span class="o">=</span> <span class="nn">Lasso</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span>
478    <span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span>
479    <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>
480    <span class="nn">LassoParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span><span class="nf">.with_alpha</span><span class="p">(</span><span class="mf">0.5</span><span class="p">),</span>
481<span class="p">)</span>
482<span class="nf">.and_then</span><span class="p">(|</span><span class="n">lr</span><span class="p">|</span> <span class="n">lr</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span>
483<span class="nf">.unwrap</span><span class="p">();</span>
484<span class="c1">// Calculate test error</span>
485<span class="nd">println!</span><span class="p">(</span><span class="s">"MSE LASSO: {}"</span><span class="p">,</span> <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_lasso</span><span class="p">));</span>
486</code></pre></div></div>
487
488<h4 id="elastic-net">Elastic Net</h4>
489
490<p>Elastic net linear regression uses the penalties from both the lasso and ridge techniques to regularize regression models.</p>
491
492<p>\[\lambda_1 \sum_{i=i}^n \beta_i^2 + \lambda_2 \sum_{i=i}^n \mid \beta_i \mid\]</p>
493
494<p>where \(\lambda_1 = \alpha l_{1r}\), \(\lambda_2 = \alpha (1 -  l_{1r})\) and \(l_{1r}\) is the l1 ratio, elastic net mixing parameter.</p>
495
496<p>elastic net combines both the L1 and L2 penalties during training, which can result in better performance than a model with either one or the other penalty on some problems.</p>
497
498<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
499<span class="c1">// DenseMatrix wrapper around Vec</span>
500<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
501<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linear</span><span class="p">::</span><span class="nn">elastic_net</span><span class="p">::{</span><span class="n">ElasticNet</span><span class="p">,</span> <span class="n">ElasticNetParameters</span><span class="p">};</span>
502<span class="c1">// Model performance</span>
503<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
504<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
505<span class="c1">// Load dataset</span>
506<span class="k">let</span> <span class="n">boston_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
507<span class="c1">// Transform dataset into a NxM matrix</span>
508<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">boston_data</span><span class="py">.num_samples</span><span class="p">)</span>
509    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">boston_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
510    <span class="nf">.collect</span><span class="p">();</span>
511<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
512<span class="c1">// These are our target values</span>
513<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">boston_data</span><span class="py">.target</span><span class="p">;</span>
514<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
514Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
515<span class="c1">// Elastic Net</span>
516<span class="k">let</span> <span class="n">y_hat_en</span> <span class="o">=</span> <span class="nn">ElasticNet</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span>
517    <span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span>
518    <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>
519    <span class="nn">ElasticNetParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span>
520        <span class="nf">.with_alpha</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span>
521        <span class="nf">.with_l1_ratio</span><span class="p">(</span><span class="mf">0.5</span><span class="p">),</span>
522<span class="p">)</span>
523<span class="nf">.and_then</span><span class="p">(|</span><span class="n">lr</span><span class="p">|</span> <span class="n">lr</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span>
524<span class="nf">.unwrap</span><span class="p">();</span>
525<span class="c1">// Calculate test error</span>
526<span class="nd">println!</span><span class="p">(</span>
527    <span class="s">"MSE Elastic Net: {}"</span><span class="p">,</span>
528    <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_en</span><span class="p">)</span>
529<span class="p">);</span>
530</code></pre></div></div>
531
532<h3 id="logistic-regression">Logistic Regression</h3>
533
534<p>Logistic regression uses linear model to represent relashionship between dependent and explanatory variables. Unlike linear regression, output in logistic regression is modeled as a binary value (0 or 1) rather than a numeric value. to squish output between 0 and 1 <a href="https://en.wikipedia.org/wiki/Sigmoid_function">Sigmoid function</a> is used.</p>
535
536<p>In <em>SmartCore</em> Logistic Regression is represented by <a href="https://docs.rs/smartcore/0.6/smartcore/linear/logistic_regression/index.html"><code class="language-plaintext highlighter-rouge">LogisticRegression</code></a> struct that has methods <code class="language-plaintext highlighter-rouge">fit</code> and <code class="language-plaintext highlighter-rouge">predict</code>.</p>
537
538<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
539<span class="c1">// DenseMatrix wrapper around Vec</span>
540<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
541<span class="c1">// Logistic Regression</span>
542<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linear</span><span class="p">::</span><span class="nn">logistic_regression</span><span class="p">::</span><span class="n">LogisticRegression</span><span class="p">;</span>
543<span class="c1">// Model performance</span>
544<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">accuracy</span><span class="p">;</span>
545<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
546<span class="c1">// Load dataset</span>
547<span class="k">let</span> <span class="n">cancer_data</span> <span class="o">=</span> <span class="nn">breast_cancer</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
548<span class="c1">// Transform dataset into a NxM matrix</span>
549<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">cancer_data</span><span class="py">.num_samples</span><span class="p">)</span>
550    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">cancer_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
551    <span class="nf">.collect</span><span class="p">();</span>
552<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
553<span class="c1">// These are our target class labels</span>
554<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">cancer_data</span><span class="py">.target</span><span class="p">;</span>
555<span class="c1">// Split dataset into training/test (80%/20%)</span>
556<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
556Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
557<span class="c1">// Logistic Regression</span>
558<span class="k">let</span> <span class="n">y_hat_lr</span> <span class="o">=</span> <span class="nn">LogisticRegression</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span> <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">())</span>
559    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">lr</span><span class="p">|</span> <span class="n">lr</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>
560<span class="c1">// Calculate test error</span>
561<span class="nd">println!</span><span class="p">(</span><span class="s">"accuracy: {}"</span><span class="p">,</span> <span class="nf">accuracy</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_lr</span><span class="p">));</span>
562</code></pre></div></div>
563
564<p><em>SmartCore</em> uses <a href="https://en.wikipedia.org/wiki/Limited-memory_BFGS">Limited-memory BFGS</a> routine to find optimal combination of \(\beta_i\) parameters.</p>
565
566<h2 id="support-vector-machines">Support Vector Machines</h2>
567
568<p>Support Vector Machines (SVM) is perhaps one of the most popular machine learning algorithms. SVMs have been shown to perform well in a variety of settings, and is often considered one of the best “out of the box” classifiers. The support vector machines is a generalization of a simple and intuitive classifier called the maximal margin classifier.</p>
569
570<p>The maximal margin classifier is a hypothetical classifier that best explains how SVM works in practice. This classifier is based on the idea of a hyperplane, a flat affine subspace of dimension \(p-1\) that divides p-dimensional space into two halves. A hyperplane is defined by the equation</p>
571
572<p>\[\beta_0 + \beta_1X_1 + \beta_2X_2 + … + \beta_pX_p = 0\]</p>
573
574<p>To classify a new point using this line we plug in input values into this equation, and check on which side of the hyperplane a point lies by calculating the sign of the left hand side of it. 
575When the equation returns a value greater than 0 the point belongs to the first class, when the equation returns a value less than 0 and the point belongs to the second class.</p>
576
577<p>The distance between the hyperplane and the closest data points is referred to as the margin. The best or optimal hyperplane that can separate the two classes is the hyperplane that has the largest margin.
578This is called the maximal margin hyperplane.</p>
579
580<p>In practice, real data is messy and cannot be separated perfectly with a hyperplane. The generalization of the maximal margin classifier to the non-separable case is known as the support vector classifier.</p>
581
582<p>The support vector classifier (SVC) is an extension of the maximal margin classifier that results from enlarging the feature space in a specific way, using kernels. SVC allow some observations to be on the incorrect side of the margin, or even the incorrect side of the hyperplane rather than seeking the largest possible margin so that every observation is on the correct side of the hyperplane.</p>
583
584<figure class="image" align="center">
585  <img src="/assets/imgs/linear_svm.svg" alt="Simple linear regression" class="img-fluid" />
586  <figcaption>Figure 2. Linear decision boundary of the Support Vector Classifier.</figcaption>
587</figure>
588
589<p>As maximal margin classifier, the SVC classifies a test observation depending on which side of a hyperplane it lies. The hyperplane is chosen to correctly separate most of the training observations into the two classes, but may misclassify a few observations. It is the solution to the optimization problem:</p>
590
591<p>\[\underset{\beta_1, \beta_2, …, \beta_p, \epsilon_1, …, \epsilon_n, M}{maximize} \space \space M \]</p>
592
593<p>subject to:
594\[\sum_{j=1}^p\beta_j^2  = 1\]
595\[y_i(\beta_0 + \beta_1x_{i1} + \beta_2x_{i2} + … + \beta_px_{ip}) \geq M(1 - \epsilon_i) \]
596\[\epsilon_i \geq 0, \sum_{i=1}^n \epsilon_i \leq C\]</p>
597
598<p>where C is a nonnegative tuning parameter, M is the width of the margin and \(\epsilon_1, …, \epsilon_n\) are slack variables that allow individual observations to be on the wrong side of the margin or the hyperplane.</p>
599
600<p>C controls the bias-variance trade-off of the support vector classifier. When the tuning parameter C is large, then the margin is wide, many observations violate the margin, and so there are many supp
600ort vectors. If C is small, then there will be fewer support vectors and hence the resulting classifier will have low bias but high variance.</p>
601
602<p>The solution to the support vector classifier optimization problem involves only the inner products of the observations, rather than the observations themselves. When we replace the inner product with a generalization that defines the similarity between new data and the support vectors the resulting classifier is known as a support vector machine. The similarity between a data point and the support vectors is called the kernel function.</p>
603
604<figure class="image" align="center">
605  <img src="/assets/imgs/rbf_svm.svg" alt="Simple linear regression" class="img-fluid" />
606  <figcaption>Figure 3. Support Vector Classifier with RBF kernel.</figcaption>
607</figure>
608
609<p><em>SmartCore</em> supports multiple kernel functions but you can always define a new kernel function by implementing the <a href="https://docs.rs/smartcore/0.6/smartcore/svm/trait.Kernel.html"><code class="language-plaintext highlighter-rouge">Kernel</code></a> trait. Not all functions can be a kernel.
610Building a new kernel requires a good mathematical understanding of the <a href="https://en.wikipedia.org/wiki/Mercer%27s_theorem">Mercer theorem</a>
611that gives necessary and sufficient condition for a function to be a kernel function.</p>
612
613<p>Pre-defined kernel functions:</p>
614
615<table class="table table-striped table-bordered">
616  <thead>
617    <tr>
618      <th>Kernel</th>
619      <th>Description</th>
620    </tr>
621  </thead>
622  <tbody>
623    <tr>
624      <td>Linear</td>
625      <td>\( K(x, x’) = \langle x, x’ \rangle\)</td>
626    </tr>
627    <tr>
628      <td>Polynomial</td>
629      <td>\( K(x, x’) = (\gamma\langle x, x’ \rangle + r)^d\), where \(d\) is polynomial degree, \(\gamma\) is a kernel coefficient and \(r\) is an independent term in the kernel function.</td>
630    </tr>
631    <tr>
632      <td>RBF (Gaussian)</td>
633      <td>\( K(x, x’) = e^{-\gamma \lVert x - x’ \rVert ^2} \), where \(\gamma\) is kernel coefficient</td>
634    </tr>
635    <tr>
636      <td>Sigmoid (hyperbolic tangent)</td>
637      <td>\( K(x, x’) = \tanh ( \gamma \langle x, x’ \rangle + r ) \), where \(\gamma\) is kernel coefficient and \(r\) is an independent term in the kernel function.</td>
638    </tr>
639  </tbody>
640</table>
641
642<h3 id="support-vector-classifier">Support Vector Classifier</h3>
643
644<p>To fit a support vector classifier to your dataset use <a href="https://docs.rs/smartcore/0.6/smartcore/svm/svc/index.html"><code class="language-plaintext highlighter-rouge">SVC</code></a>. 
645<em>SmartCore</em> uses an <a href="https://leon.bottou.org/projects/lasvm">approximate SVM solver</a> to solve SMV optimization problem. 
646The solver reaches accuracies similar to that of a real SVM after performing two passes through the training examples. 
647You can choose the number of passes through the data that the algorithm takes by changing the <code class="language-plaintext highlighter-rouge">epoch</code> parameter of the classifier.</p>
648
649<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
650<span class="c1">// DenseMatrix wrapper around Vec</span>
651<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
652<span class="c1">// SVM</span>
653<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">svm</span><span class="p">::</span><span class="nn">svc</span><span class="p">::{</span><span class="n">SVCParameters</span><span class="p">,</span> <span class="n">SVC</span><span class="p">};</span>
654<span class="c1">// Model performance</span>
655<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">accuracy</span><span class="p">;</span>
656<span class="k">
656use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
657<span class="c1">// Load dataset</span>
658<span class="k">let</span> <span class="n">cancer_data</span> <span class="o">=</span> <span class="nn">breast_cancer</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
659<span class="c1">// Transform dataset into a NxM matrix</span>
660<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">cancer_data</span><span class="py">.num_samples</span><span class="p">)</span>
661    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">cancer_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
662    <span class="nf">.collect</span><span class="p">();</span>
663<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
664<span class="c1">// These are our target class labels</span>
665<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">cancer_data</span><span class="py">.target</span><span class="p">;</span>
666<span class="c1">// Split dataset into training/test (80%/20%)</span>
667<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
668<span class="c1">// SVC — note: SVC borrows its parameters (implements SupervisedEstimatorBorrow),</span>
669<span class="c1">// so pass &amp;SVCParameters, not an owned value</span>
670<span class="k">let</span> <span class="n">y_hat_svm</span> <span class="o">=</span> <span class="nn">SVC</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="nn">SVCParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span><span class="nf">.with_c</span><span class="p">(</span><span class="mf">10.0</span><span class="p">))</span>
671    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">svm</span><span class="p">|</span> <span class="n">svm</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span>
672    <span class="nf">.unwrap</span><span class="p">();</span>
673<span class="c1">// Calculate test error</span>
674<span class="nd">println!</span><span class="p">(</span><span class="s">"accuracy SVM: {}"</span><span class="p">,</span> <span class="nf">accuracy</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_svm</span><span class="p">));</span>
675</code></pre></div></div>
676
677<h3 id="support-vector-regressor">Support Vector Regressor</h3>
678
679<p>To fit support vector regressor to your dataset use <a href="https://docs.rs/smartcore/0.6/smartcore/svm/svr/index.html"><code class="language-plaintext highlighter-rouge">epsilon-support SVR</code></a>.</p>
680
681<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">
681use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
682<span class="c1">// DenseMatrix wrapper around Vec</span>
683<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
684<span class="c1">// SVM</span>
685<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">svm</span><span class="p">::</span><span class="nn">svr</span><span class="p">::{</span><span class="n">SVRParameters</span><span class="p">,</span> <span class="n">SVR</span><span class="p">};</span>
686<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">svm</span><span class="p">::</span><span class="n">Kernels</span><span class="p">;</span>
687<span class="c1">// Model performance</span>
688<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
689<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
690<span class="c1">// Load dataset — boston is a regression dataset (target Vec&lt;f32&gt;).</span>
691<span class="c1">// Note: `diabetes::load_dataset()` returns u32 targets which are not</span>
692<span class="c1">// `FloatNumber`, so it cannot be used with SVR.</span>
693<span class="k">let</span> <span class="n">diabetes_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
694<span class="c1">// Transform dataset into a NxM matrix</span>
695<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">diabetes_data</span><span class="py">.num_samples</span><span class="p">)</span>
696    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">diabetes_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">diabetes_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">diabetes_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
697    <span class="nf">.collect</span><span class="p">();</span>
698<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
699<span class="c1">// These are our target values</span>
700<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">diabetes_data</span><span class="py">.target</span><span class="p">;</span>
701<span class="c1">// Split dataset into training/test (80%/20%)</span>
702<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">
702Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
703<span class="c1">// SVR — like SVC, SVR borrows its parameters</span>
704<span class="k">let</span> <span class="n">y_hat_svm</span> <span class="o">=</span> <span class="nn">SVR</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span>
705    <span class="o">&amp;</span><span class="nn">SVRParameters</span><span class="p">::</span><span class="nf">default</span><span class="p">()</span><span class="nf">.with_kernel</span><span class="p">(</span><span class="nn">Kernels</span><span class="p">::</span><span class="nf">rbf</span><span class="p">(</span><span class="mf">0.5</span><span class="p">))</span><span class="nf">.
706        with_c</span><span class="p">(</span><span class="mf">2000.0</span><span class="p">)</span><span class="nf">.with_eps</span><span class="p">(</span><span class="mf">10.0</span><span class="p">))</span>
707    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">svm</span><span class="p">|</span> <span class="n">svm</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span>
708    <span class="nf">.unwrap</span><span class="p">();</span>
709<span class="c1">// Calculate test error</span>
710<span class="nd">println!</span><span class="p">(</span>
711    <span class="s">"MSE: {}"</span><span class="p">,</span>
712    <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_svm</span><span class="p">)</span>
713<span class="p">);</span>
714</code></pre></div></div>
715
716<h2 id="naive-bayes">Naive Bayes</h2>
717
718<p>Naive Bayes (NB) is a probabilistic machine learning algorithm based on the Bayes Theorem that assumes conditional independence between features given the value of the class variable.</p>
719
720<p>Bayes Theorem states following relashionship between class label and data:</p>
721
722<p>\[ P(y \mid X) = \frac{P(y)P(X \mid y)}{P(X)} \]</p>
723
724<p>where</p>
725<ul>
726  <li>\(X = (x_1,…x_n)\) represents the predictors.</li>
727  <li>\(P(y \mid X)\) is the probability of class <em>y</em> given the data X</li>
728  <li>\(P(X \mid y)\) is the probability of data X given the class <em>y</em>.</li>
729  <li>\(P(y)\) is the probability of class y. This is called the prior probability of y.</li>
730  <li>\(P(y \mid X)\) is the probability of the data (regardless of the class value).</li>
731</ul>
732
733<p>We are interested in calculating the posterior probability of \(P(y \mid X)\) from the prior probability \(P(y)\), conditional probability \(P(X \mid y)\) and \(P(X)\). 
734The naive conditional independence assumption let us rewrite this equation as</p>
735
736<p>\[ P(y \mid x_1,…x_n) = \frac{P(y)\prod_{i=1}^nP(x_i \mid y)}{P(x_1,…x_n)} \]</p>
737
738<p>The denominator can be removed since \(P(x_1,…x_n)\) is constant for all the entries in the dataset.</p>
739
740<p>\[ P(y \mid x_1,…x_n) \propto P(y)\prod_{i=1}^nP(x_i \mid y) \]</p>
741
742<p>To find class y from predictors X we use this equation</p>
743
744<p>\[ y = \underset{y}{argmax} P(y)\prod_{i=1}^nP(x_i \mid y) \]</p>
745
746<p>Specific variants of the Naive Bayes classifier have different assumptions regarding the distribution of \(P(x_i \mid y)\). 
747The table below displays variants of Naive Bayes classifiers implemented in <em>SmartCore</em>:</p>
748
749<table class="table table-striped table-bordered">
750  <thead>
751    <tr>
752      <th>Name</th>
753      <th>The assumptions on distribution of features</th>
754    </tr>
755  </thead>
756  <tbody>
757    <tr>
758      <td><a href="https://docs.rs/smartcore/0.6/smartcore/naive_bayes/bernoulli/index.html">Bernoulli NB</a></td>
759      <td>features are independent binary variables</td>
760    </tr>
761    <tr>
762      <td><a href="https://docs.rs/smartcore/0.6/smartcore/naive_bayes/multinomial/index.html">Multinomial NB</a></td>
763      <td>features are the frequencies with which certain events have been generated by a multinomial distribution</td>
764    </tr>
765    <tr>
766      <td><a href="https://docs.rs/smartcore/0.6/smartcore/naive_bayes/categorical/index.html">Categorical NB</a></td>
767      <td>each feature has its own categorical distribution</td>
768    </tr>
769    <tr>
770      <td><a href="https://docs.rs/smartcore/0.6/smartcore/naive_bayes/gaussian/index.html">Gaussian NB</a></td>
771      <td>continuous data distributed according to a Gaussian distribution</td>
772    </tr>
773  </tbody>
774</table>
775
776<p>For example, this is how you can fit Gaussian NB to the Iris dataset:</p>
777
778<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="nn">iris</span><span class="p">::</span><span class="n">load_dataset</span><span class="p">;</span>
779<span class="c1">// DenseMatrix wrapper around Vec</span>
780<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
781<span class="c1">// Imports Gaussian Naive Bayes classifier</span>
782<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">naive_bayes</span><span class="p">::</span><span class="nn">gaussian</span><span class="p">::</span><span class="n">GaussianNB</span><span class="p">;</span>
783<span class="c1">// Model performance</span>
784<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">accuracy</span><span class="p">;</span>
785<span class="c1">// Load Iris dataset</span>
786<span class="k">let</span> <span class="n">iris_data</span> <span class="o">=</span> <span class="nf">load_dataset</span><span class="p">();</span>
787<span class="c1">// Turn Iris dataset into NxM matrix</span>
788<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">iris_data</span><span class="py">.num_samples</span><span class="p">)</span>
789    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">iris_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">iris_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">iris_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
790    <span class="nf">.collect</span><span class="p">();</span>
791<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
792<span class="c1">// These are our target class labels</span>
793<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">iris_data</span><span class="py">.target</span><span class="p">;</span>
794<span class="c1">// Fit Gaussian Naive Bayes to Iris dataset</span>
795<span class="k">let</span> <span class="n">gnb</span> <span class="o">=</span> <span class="nn">GaussianNB</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">())</span><span class="nf">.unwrap</span><span class="p">();</span>
796<span class="k">let</span> <span class="n">y_hat</span> <span class="o">=</span> <span class="n">gnb</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span> <span class="c1">// Predict class labels</span>
797<span class="c1">// Calculate training error</span>
798<span class="nd">println!</span><span class="p">(</span><span class="s">"accuracy: {}"</span><span class="p">,</span> <span class="nf">accuracy</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat</span><span class="p">));</span> <span class="c1">// Prints 0.96</span>
799</code></pre></div></div>
800
801<h2 id="decision-trees">Decision Trees</h2>
802
803<p>Classification and Regression Trees (CART) and its modern variant Random Forest are among the most powerful algorithms available in machine learning.</p>
804
805<p>CART models relationship between predictor and explanatory variables as a binary tree. Each node of the tree represents a decision that is made based on an outcome of a single attribute.
806The leaf nodes of the tree represent an outcome. To make a prediction we take the mean of the training observations belonging to the leaf node for regression and the mode of observations for classification.</p>
807
808<p>Given a dataset with just three explanatory variables and a qualitative dependent variable the tree might look like an example below.</p>
809
810<figure class="image" align="center">
811  <img src="/assets/imgs/tree.svg" alt="Decision Tree example" class="img-fluid" />
812  <figcaption>Figure 4. An example of Decision Tree where target is a class.</figcaption>
813</figure>
814
815<p>CART model is simple and useful for interpretation. However, they typically are not competitive with the best supervised learning approaches, like Logistic and Linear Regression, especially when the response can be well approximated by a linear model. Tree-based method is also non-robust which means that a small change in the data can cause a large change in the final estimated tree. That’s why it is a common practice to combine prediction from multiple trees in ensemble to estimate predicted values.</p>
816
817<p>In <em>SmartCore</em> both, decision and regression trees can be found in the <a href="https://docs.rs/smartcore/0.6/smartcore/tree/index.html"><code class="language-plaintext highlighter-rouge">tree</code></a> module. Use <a href="https://docs.rs/smartcore/0.6/smartcore/tree/decision_tree_classifier/index.html"><code class="language-plaintext highlighter-rouge">DecisionTreeClassifier</code></a> to fit decision tree and <a href="https://docs.rs/smartcore/0.6/smartcore/tree/decision_tree_regressor/index.html"><code class="language-plaintext highlighter-rouge">DecisionTreeRegressor</code></a> for regression.</p>
818
819<p>To fit <a href="https://docs.rs/smartcore/0.6/smartcore/tree/decision_tree_classifier/index.html"><code class="language-plaintext highlighter-rouge">DecisionTreeClassifier</code></a> to <a href="https://docs.rs/smartcore/0.6/smartcore/dataset/breast_cancer/index.html">Breast Cancer</a> dataset:</p>
820
821<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">
821use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
822<span class="c1">// DenseMatrix wrapper around Vec</span>
823<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
824<span class="c1">// Tree</span>
825<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">tree</span><span class="p">::</span><span class="nn">decision_tree_classifier</span><span class="p">::</span><span class="n">DecisionTreeClassifier</span><span class="p">;</span>
826<span class="c1">// Model performance</span>
827<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">accuracy</span><span class="p">;</span>
828<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
829<span class="c1">// Load dataset</span>
830<span class="k">let</span> <span class="n">cancer_data</span> <span class="o">=</span> <span class="nn">breast_cancer</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
831<span class="c1">// Transform dataset into a NxM matrix</span>
832<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">cancer_data</span><span class="py">.num_samples</span><span class="p">)</span>
833    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">cancer_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">cancer_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
834    <span class="nf">.collect</span><span class="p">();</span>
835<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
836<span class="c1">// These are our target class labels</span>
837<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">cancer_data</span><span class="py">.target</span><span class="p">;</span>
838<span class="c1">// Split dataset into training/test (80%/20%)</span>
839<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
840<span class="c1">// Decision Tree</span>
841<span class="k">let</span> <span class="n">y_hat_tree</span> <span class="o">=</span> <span class="nn">DecisionTreeClassifier</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span> <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">())</span>
842    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">tree</span><span class="p">|</span> <span class="n">tree</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>
843<span class="c1">// Calculate test error</span>
844<span class="nd">println!</span><span class="p">(</span><span class="s">"accuracy: {}"</span><span class="p">,</span> <span class="nf">accuracy</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_tree</span><span class="p">));</span>
845</code></pre></div></div>
846
847<p>Here we have used default parameter values but in practice you will almost always use <a href="https://en.wikipedia.org/wiki/Cross-validation_(statistics)">k-fold cross validation</a> or hold-out validation dataset to fine tune your parameter values.</p>
848
849<h2 id="ensemble-methods">Ensemble methods</h2>
850
851<p>In ensemble learning we combine predictions from multiple base models to reduce the variance of predictions and decrease generalization error. Base models are assumed to be independent from each other. <a href="https://en.wikipedia.org/wiki/Bootstrap_aggregating">Bagging</a> is one of the most streightforward ways to reduce correlation between base models in the ensemble. It works by taking repeated samples from the same training data set. As a result we generate <em>K</em> different training data sets (bootstraps) that overlap but are not the same. We then train our base model on the each bootstrapped training set and average predictions for regression or use majority voting scheme for classification.</p>
852
853<h3 id="random-forest">Random Forest</h3>
854
855<p>Random forest is an extension of bagging that also randomly selects a subset of features when training a tree. This improvement decorrelated the trees and hence decreases prediction error even more. Random forests have proven effective on a wide range of different predictive modeling problems.</p>
856
857<p>Let’s fit <a href="https://docs.rs/smartcore/0.6/smartcore/ensemble/random_forest_regressor/index.html">Random Forest regressor</a> to Boston Housing dataset:</p>
858
859<div class="language-rust highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="k">
859use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">dataset</span><span class="p">::</span><span class="o">*</span><span class="p">;</span>
860<span class="c1">// DenseMatrix wrapper around Vec</span>
861<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">linalg</span><span class="p">::</span><span class="nn">basic</span><span class="p">::</span><span class="nn">matrix</span><span class="p">::</span><span class="n">DenseMatrix</span><span class="p">;</span>
862<span class="c1">// Random Forest</span>
863<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">ensemble</span><span class="p">::</span><span class="nn">random_forest_regressor</span><span class="p">::</span><span class="n">RandomForestRegressor</span><span class="p">;</span>
864<span class="c1">// Model performance</span>
865<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">metrics</span><span class="p">::</span><span class="n">mean_squared_error</span><span class="p">;</span>
866<span class="k">use</span> <span class="nn">smartcore</span><span class="p">::</span><span class="nn">model_selection</span><span class="p">::</span><span class="n">train_test_split</span><span class="p">;</span>
867<span class="c1">// Load dataset</span>
868<span class="k">let</span> <span class="n">boston_data</span> <span class="o">=</span> <span class="nn">boston</span><span class="p">::</span><span class="nf">load_dataset</span><span class="p">();</span>
869<span class="c1">// Transform dataset into a NxM matrix</span>
870<span class="k">let</span> <span class="n">rows</span><span class="p">:</span> <span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">Vec</span><span class="o">&lt;</span><span class="nb">f32</span><span class="o">&gt;&gt;</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="o">..</span><span class="n">boston_data</span><span class="py">.num_samples</span><span class="p">)</span>
871    <span class="nf">.map</span><span class="p">(|</span><span class="n">r</span><span class="p">|</span> <span class="n">boston_data</span><span class="py">.data</span><span class="p">[</span><span class="n">r</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="o">..</span><span class="p">(</span><span class="n">r</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">boston_data</span><span class="py">.num_features</span><span class="p">]</span><span class="nf">.to_vec</span><span class="p">())</span>
872    <span class="nf">.collect</span><span class="p">();</span>
873<span class="k">let</span> <span class="n">x</span> <span class="o">=</span> <span class="nn">DenseMatrix</span><span class="p">::</span><span class="nf">from_2d_vec</span><span class="p">(</span><span class="o">&amp;</span><span class="n">rows</span><span class="p">)</span><span class="nf">.unwrap</span><span class="p">();</span>
874<span class="c1">// These are our target class labels</span>
875<span class="k">let</span> <span class="n">y</span> <span class="o">=</span> <span class="n">boston_data</span><span class="py">.target</span><span class="p">;</span>
876<span class="c1">// Split dataset into training/test (80%/20%)</span>
877<span class="k">let</span> <span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span> <span class="o">=</span> <span class="nf">train_test_split</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="k">true</span><span class="p">,</span> <span class="nf">Some</span><span class="p">(</span><span class="mi">42</span><span class="p">));</span>
878<span class="c1">// Random Forest</span>
879<span class="k">let</span> <span class="n">y_hat_rf</span> <span class="o">=</span> <span class="nn">RandomForestRegressor</span><span class="p">::</span><span class="nf">fit</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_train</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_train</span><span class="p">,</span> <span class="nn">Default</span><span class="p">::</span><span class="nf">default</span><span class="p">())</span>
880    <span class="nf">.and_then</span><span class="p">(|</span><span class="n">
880rf</span><span class="p">|</span> <span class="n">rf</span><span class="nf">.predict</span><span class="p">(</span><span class="o">&amp;</span><span class="n">x_test</span><span class="p">))</span><span class="nf">.unwrap</span><span class="p">();</span>
881<span class="c1">// Calculate test error</span>
882<span class="nd">println!</span><span class="p">(</span><span class="s">"MSE: {}"</span><span class="p">,</span> <span class="nf">mean_squared_error</span><span class="p">(</span><span class="o">&amp;</span><span class="n">y_test</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">y_hat_rf</span><span class="p">));</span>
883</code></pre></div></div>
884
885<p>You should get lower mean squared error here when compared to other methods from this manual. This is because by default Random Forest fits 100 independent trees to different bootstrapped training sets and calculates target value by averaging predictions from these trees.</p>
886
887<p><a href="https://docs.rs/smartcore/0.6/smartcore/ensemble/random_forest_classifier/index.html">Random Forest classifier</a> works in a similar manner. The only difference is that you prediction targets should be nominal or ordinal values (class label).</p>
888
889<h2 id="references">References</h2>
890<ul>
891  <li><a href="http://ssg.mit.edu/cal/abs/2000_spring/np_dens/classification/cover67.pdf">“Nearest Neighbor Pattern Classification” Cover, T.M., IEEE Transactions on Information Theory (1967)</a></li>
892  <li><a href="https://web.stanford.edu/~hastie/ElemStatLearn/">“The Elements of Statistical Learning: Data Mining, Inference, and Prediction” Trevor et al., 2nd edition</a></li>
893  <li><a href="http://faculty.marshall.usc.edu/gareth-james/ISL/">“An Introduction to Statistical Learning”, James G., Witten D., Hastie T., Tibshirani R.</a></li>
894  <li><a href="https://www-cs-faculty.stanford.edu/~knuth/taocp.html">“The Art of Computer Programming” Knuth, D, Vol. 3, 2nd ed, Sorting and Searching, 1998</a></li>
895  <li><a href="https://mitpress.mit.edu/books/machine-learning-1">“Machine Learning: A Probabilistic Perspective”, Kevin P. Murphy, 2012, Chapter 3 </a></li>
896  <li><a href="https://hunch.net/~jl/projects/cover_tree/cover_tree.html">“Cover Trees for Nearest Neighbor” Beygelzimer et al., Proceedings of the 23rd international conference on Machine learning, ICML’06 (2006)</a></li>
897  <li><a href="http://www.cs.ucr.edu/~cshelton/papers/index.cgi%3FIzbShe15">“Faster cover trees.” Izbicki et al., Proceedings of the 32nd International Conference on Machine Learning, ICML’15 (2015)</a></li>
898  <li><a href="http://numerical.recipes/">“Numerical Recipes: The Art of Scientific Computing”,  Press W.H., Teukolsky S.A., Vetterling W.T, Flannery B.P, 3rd ed.</a></li>
899  <li><a href="https://www.microsoft.com/en-us/research/uploads/prod/2006/01/Bishop-Pattern-Recognition-and-Machine-Learning-2006.pdf">“Pattern Recognition and Machine Learning”, C.M. Bishop, Linear Models for Classification</a></li>
900  <li><a href="http://users.iems.northwestern.edu/~nocedal/PDFfiles/limited.pdf">“On the Limited Memory Method for Large Scale Optimization”, Nocedal et al., Mathematical Programming, 1989</a></li>
901  <li><a href="https://www.sciencebase.gov/catalog/item/545d07dfe4b0ba8303f728c1">“Classification and regression trees”, Breiman, L, Friedman, J H, Olshen, R A, and Stone, C J, 1984</a></li>
902  <li><a href="https://www.svm-tutorial.com/2017/10/support-vector-machines-succinctly-released/">“Support Vector Machines”, Kowalczyk A., 2017</a></li>
903</ul>
904
905                
906                <footer class="my-md-5 pt-md-5 border-top">    
907    <div class="row justify-content-md-center">
908        <div class="col-md-auto">
909            <div class="row">
910                <img class="mr-1" alt="License Apache 2.0" src="https://img.shields.io/github/license/smartcorelib/smartcore">
911                <img class="mr-1" alt="Build status" src="https://img.shields.io/circleci/build/github/smartcorelib/smartcore">
912                <img class="mr-1" alt="Code coverage" src="https://img.shields.io/codecov/c/github/smartcorelib/smartcore">
913                <img class="mr-1" alt="Latest version" src="https://img.shields.io/crates/v/smartcore">            
914            </div>
915        </div>
916        <div class="col">
917            <div class="d-block mb-3 text-muted text-right">
918                MIT/APACHE-2.0 © 2020-2026 SmartCore developers            
919            </div>             
920        </div>  
921    </div>      
922</footer>
923
924            </div>
925        </div>        
926    </div>    
927    
927<script src="https://code.jquery.com/jquery-3.5.1.slim.min.js" integrity="sha384-DfXdz2htPH0lsSSs5nCTpuj/zy4C+OGpamoFVy38MVBnE+IbbVYUew+OrCXaRkfj" crossorigin="anonymous"></script>
927
928    
928<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/umd/popper.min.js" integrity="sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN" crossorigin="anonymous"></script>
928
929    
929<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js" integrity="sha384-B4gt1jrGC7Jh4AgTPSdUtOBvfO8shuf57BaghqFfPlYxofvL8/KUEfYiJOMMV+rV" crossorigin="anonymous"></script>
929
930    
930<script src="https://cdn.rawgit.com/afeld/bootstrap-toc/v1.0.1/dist/bootstrap-toc.min.js"></script>
930    
931    
931<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
931
932    
932<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
932
933    
933<script src="/assets/js/main.js"></script>
933    
934  </body>
935</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.