PageSourceSearch

https://investigate.ai/wapo-app-reviews/scrape-app-store-reviews/

html investigate.ai collected 2026-10-03 21:37:12 UTC 80,242 bytes, 2,895 lines download raw bytes

1<!doctype html>
2<html lang="en-US">
3
4
5
6<head>
7  <!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 66 bytes, lines 7-8
7
8  <script async src="https://www.googletagmanager.com/gtag/js?id=
8UA-5541738-27
vendor: 14 bytes, lines 8-9
8"></script>
9  
9<script>
10    
vendor: 140 bytes, lines 10-14
10window.dataLayer = window.dataLayer || [];
11    function gtag(){dataLayer.push(arguments);}
12    gtag('js', new Date());
13
14    gtag('config', '
14UA-5541738-27
vendor: 6 bytes, lines 14-15
14');
15  
15</script>
15
16
17  <meta charset="utf-8">
18  <title>Scrape and combine app store reviews, from random chat app safety by WaPo</title>
19  <link href="/font-awesome/css/all.min.css?v=500d1a92f875b1d96d37a3a3f8f0438c" rel="stylesheet">
20  <!-- <link href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.5/css/bulma.min.css" rel="stylesheet">
21        <link href="https://fonts.googleapis.com/css?family=Raleway:400,700|Open+Sans:400,700&display=swap" rel="stylesheet"> -->
22  <link rel="stylesheet" href="/css/spectre.min.css?v=5cd401d486f79e82913923fe7d7f47ff">
23  <link rel="stylesheet" href="/css/spectre-exp.min.css?v=5909d80638a6ae6aa3a455b6f6a6d768">
24  <link rel="stylesheet" href="/css/spectre-icons.min.css?v=56b1bd38b79450b37939f8adb811d4cd">
25
26  <link rel="stylesheet" href="/css/tocbot.css?v=e8f0173e7c5216e5359587a88a570b77">
27
28  <!-- 
28<script src="https://unpkg.com/lunr/lunr.js"></script>
28 -->
29  
29<script src="/js/lunr.js?v=b93de9e9609074d666f5123e866e25ed"></script>
29
30  <!-- 
30<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>
30 -->
31  
31<script src="/js/require.min.js?v=18823f6a6d208ee1e361bb266ab794d5"></script>
31
32
33  <meta name="viewport" content="width=device-width, initial-scale=1">
34  <link href="/css/style.css?v=10420579fc48bad7c182d7d5e44a2cbb" rel="stylesheet">
35  <link href="/css/highlight.css?v=bdd372d828c6988b6071bb877211ebcb" rel="stylesheet">
36
37  <meta name="description" content="Using a website devoted to phone app marketing, we'll download over 50,000 reviews for various apps and save them to a CSV. A friendly data journalism lesson.">
38  <meta content="investigate.ai: Data Science for Journalists" property="og:site_name">
39
40  <meta name="twitter:creator" content="@dangerscarf">
41  <meta name="twitter:title" content="Scrape and combine app store reviews, from random chat app safety by WaPo">
42  <meta name="twitter:description" content="Using a website devoted to phone app marketing, we'll download over 50,000 reviews for various apps and save them to a CSV. A friendly data journalism lesson.">
43  <meta name="twitter:image" content="https://investigate.ai/images/flame/flame-searching.png">
44  <meta name="twitter:image:alt" content="A person metaphorically examining data">
45  <meta name="twitter:card" content="summary" />
46
47  <meta property="og:type" content="website" />
48  <meta property="og:url" content="https://investigate.ai/wapo-app-reviews/scrape-app-store-reviews/" />
49  <meta property="og:title" content="Scrape and combine app store reviews, from random chat app safety by WaPo" />
50  <meta property="og:description" content="Using a website devoted to phone app marketing, we'll download over 50,000 reviews for various apps and save them to a CSV. A friendly data journalism lesson." />
51  <meta property="og:image" content="https://investigate.ai/images/flame/flame-searching.png" />
52  <meta property="og:image:height" content="912" />
53  <meta property="og:image:width" content="921" />
54  <meta property="fb:admins" content="1504164"/>
55
56  <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
57  <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
58  <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
59  <link rel="manifest" href="/site.webmanifest">
60  <link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">
61  <meta name="msapplication-TileColor" content="#da532c">
62  <meta name="theme-color" content="#ffffff">
63
64</head>
65
66<body class="">
67  <div>
68      <div class="content-main">
69        <div class="nav-holder bg-secondary">
70  <div class='content'>
71    <header class="navbar">
72      <a href="#" class="sidebar-opener" aria-label="Sidebar opener"><i class="fas fa-bars fa-2x"></i></a>
73      <section class="navbar-section">
74        <a href="/" class="btn btn-link">Home</a>
75        <a href="/projects/" class="btn btn-link">Projects</a>
76        <a href="/topics/" class="btn btn-link">Topics</a>
77        <a href="/search/" class="btn btn-link">Search</a>
78        <a href="/newsletter/" class="btn btn-link">Newsletter</a>
79        <a href="/about/" class="btn btn-link">About</a>
80      </section>
81    </header>
82  </div>
83</div>
84        
85<div class="content text-based">
86  <div class="columns">
87    <div class="column col-12 notebook">
88      <ul class="breadcrumb">
89        <li class="breadcrumb-item">
90          <a href="/">investigate.ai</a>
91        </li>
92        <li class="breadcrumb-item">
93          <a href="/wapo-app-reviews/">Analyzing online safety through app store reviews</a>
94        </li>
95        <li class="breadcrumb-item">
96          <a href="/wapo-app-reviews/scrape-app-store-reviews">Scrape and combine app store reviews</a>
97        </li>
98      </ul>
99
100      <div class="columns next-prev-links">
101        <div class="column col-6 col-sm-12 prev-column"></div>
102        <div class="column col-6 col-sm-12 text-right next-column"></div>
103      </div>
104
105      <div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
106<div class="text_cell_render border-box-sizing rendered_html">
107<h1 id="Scraping-app-store-reviews">Scraping app store reviews<a class="anchor-link" href="#Scraping-app-store-reviews">#</a></h1><p>In the Washington Post's project, they found a "secret API" that allowed them to download all the App Store reviews of target "random chat apps." We're going to download reviews using the marketing platform Sensor Tower instead. Our target apps will be Chat with Strangers, Yubo, Holla, and Skout.</p>
108<p>Their reviews section doesn't have a download button, so we use a Selenium web scraper to download the information instead.</p>
109
110</div>
111</div>
112</div>
113<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
114<div class="text_cell_render border-box-sizing rendered_html">
115<p class="reading-options">
116  <a class="btn" href="/wapo-app-reviews/scrape-app-store-reviews">
117    <i class="fa fa-sm fa-book"></i>
118    Read online
119  </a>
120  <a class="btn" href="/wapo-app-reviews/notebooks/Scrape app store reviews.ipynb">
121    <i class="fa fa-sm fa-download"></i>
122    Download notebook
123  </a>
124  <a class="btn" href="https://colab.research.google.com/github/littlecolumns/ds4j-notebooks/blob/master/wapo-app-reviews/notebooks/Scrape app store reviews.ipynb" target="_new">
125    <i class="fa fa-sm fa-laptop"></i>
126    Interactive version
127  </a>
128</p>
129</div>
130</div>
131</div>
132<div class="cell border-box-sizing code_cell rendered">
133<div class="input">
134
135<div class="inner_cell">
136    <div class="input_area">
137<div class=" highlight hl-ipython3"><pre><span></span><span class="kn">from</span> <span class="nn">bs4</span> <span class="kn">
137import</span> <span class="n">BeautifulSoup</span>
138<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
139<span class="kn">import</span> <span class="nn">time</span>
140<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
141
142<span class="kn">from</span> <span class="nn">selenium</span> <span class="kn">import</span> <span class="n">webdriver</span>
143<span class="kn">from</span> <span class="nn">selenium.webdriver.common.by</span> <span class="kn">import</span> <span class="n">By</span>
144<span class="kn">from</span> <span class="nn">selenium.webdriver.support.ui</span> <span class="kn">import</span> <span class="n">WebDriverWait</span>
145<span class="kn">from</span> <span class="nn">selenium.webdriver.support</span> <span class="kn">import</span> <span class="n">expected_conditions</span> <span class="k">as</span> <span class="n">EC</span>
146</pre></div>
147
148    </div>
149</div>
150</div>
151
152</div>
153<div class="cell border-box-sizing code_cell rendered">
154<div class="input">
155
156<div class="inner_cell">
157    <div class="input_area">
158<div class=" highlight hl-ipython3"><pre><span></span><span class="n">driver</span> <span class="o">=</span> <span class="n">webdriver</span><span class="o">.</span><span class="n">Chrome</span><span class="p">()</span>
159<span class="n">driver</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;https://sensortower.com/ios/US/twelve-app/app/yubo-make-new-friends/1038653883/review-history?selected_tab=reviews&#39;</span><span class="p">)</span>
160</pre></div>
161
162    </div>
163</div>
164</div>
165
166</div>
167<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
168<div class="text_cell_render border-box-sizing rendered_html">
169<h3 id="Select-your-options-and-scrape">Select your options and scrape<a class="anchor-link" href="#Select-your-options-and-scrape">#</a></h3><p>After you log in, select the following options to make sure you're only scraping US-based reviews. This is mostly to make sure we keep everything in English, as we won't be able to manually find racism etc in non-English reviews.</p>
170<ul>
171<li><strong>Date:</strong> All time</li>
172<li><strong>Country:</strong> US</li>
173</ul>
174
175</div>
176</div>
177</div>
178<div class="cell border-box-sizing code_cell rendered">
179<div class="input">
180
181<div class="inner_cell">
182    <div class="input_area">
183<div class=" highlight hl-ipython3"><pre><span></span><span class="k">def</span> <span class="nf">get_page</span><span class="p">():</span>
184    <span class="n">doc</span> <span class="o">=</span> <span class="n">BeautifulSoup</span><span class="p">(</span><span class="n">driver</span><span class="o">.</span><span class="n">page_source</span><span class="p">)</span>
185    <span class="n">rows</span> <span class="o">=</span> <span class="n">doc</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;tbody tr&quot;</span><span class="p">)</span>
186
187    <span class="n">datapoints</span> <span class="o">=</span> <span class="p">[]</span>
188    <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
189        <span class="n">cells</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="s2">&quot;td&quot;</span><span class="p">)</span>
190        <span class="n">data</span> <span class="o">=</span> <span class="p">{</span>
191            <span class="s1">&#39;Country&#39;</span><span class="p">:</span> <span class="n">cells</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span>
192            <span class="s1">&#39;Date&#39;</span><span class="p">:</span> <span class="n">cells</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span>
193            <span class="s1">&#39;Rating&#39;</span><span class="p">:</span> <span class="n">cells</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span><span class="o">.</span><span class="n">select_one</span><span class="p">(</span><span class="s1">&#39;.gold&#39;</span><span class="p">)[</span><span class="s1">&#39;style&#39;</span><span class="p">],</span>
194            <span class="s1">&#39;Review&#39;</span><span class="p">:</span> <span class="n">cells</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">select_one</span><span class="p">(</span><span class="s1">&#39;.break-wrap-review&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span>
195            <span class="s1">&#39;Version&#39;</span><span class="p">:</span> <span class="n">cells</span><span class="p">[</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
196        <span class="p">}</span>
197        <span class="n">datapoints</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
198    <span class="k">return</span> <span class="n">datapoints</span>
199
200<span class="n">all_data</span> <span class="o">=</span> <span class="p">[]</span>
201<span class="n">wait</span> <span class="o">=</span> <span class="n">WebDriverWait</span><span class="p">(</span><span class="n">driver</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">poll_frequency</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
202<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
203    <span class="n">wait</span><span class="o">.</span><span class="n">until</span><span class="p">(</span><span class="n">EC</span><span class="o">.</span><span class="n">invisibility_of_element_located</span><span class="p">((</span><span class="n">By</span><span class="o">.</span><span class="n">CSS_SELECTOR</span><span class="p">,</span> <span class="s1">&#39;.ajax-loading-cover&#39;</span><span class="p">)))</span>
204
205    <span class="n">results</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">()</span>    
206    <span class="n">all_data</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">results</span><span class="p">)</span>
207
208    <span class="n">next_button</span> <span class="o">=</span> <span class="n">driver</span><span class="o">.</span><span class="n">find_elements_by_css_selector</span><span class="p">(</span><span class="s2">&quot;.btn-group .pagination&quot;</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
209    <span class="k">if</span> <span class="n">next_button</span><span class="o">.</span><span class="n">get_attribute</span><span class="p">(</span><span class="s1">&#39;disabled&#39;</span><span class="p">):</span>
210        <span class="k">break</span>
211    <span class="n">next_button</span><span class="o">.</span><span class="n">click</span><span class="p">()</span>
212    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.5</span><span class="p">)</span>
213    <span class="c1"># Doesn&#39;t trigger fast enough!</span>
214    <span class="c1"># wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, &#39;.ajax-loading-cover&#39;)))</span>
215
216<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">all_data</span><span class="p">)</span>
217<span class="n">df</span>
218</pre></div>
219
220    </div>
221</div>
222</div>
223
224<div class="output_wrapper">
225<div class="output">
226
227<div class="output_area">
228
229
230<div class="output_html rendered_html output_subarea output_execute_result">
231<div>
232<style scoped>
233    .dataframe tbody tr th:only-of-type {
234        vertical-align: middle;
235    }
236
237    .dataframe tbody tr th {
238        vertical-align: top;
239    }
240
241    .dataframe thead th {
242        text-align: right;
243    }
244</style>
245<table border="1" class="dataframe">
246  <thead>
247    <tr style="text-align: right;">
248      <th></th>
249      <th>Country</th>
250      <th>Date</th>
251      <th>Rating</th>
252      <th>Review</th>
253      <th>Version</th>
254    </tr>
255  </thead>
256  <tbody>
257    <tr>
258      <th>0</th>
259      <td>US</td>
260      <td>11/19/2019</td>
261      <td>width: 19%;</td>
262      <td>This is an Omegle knockoff. Don’t recommend. 9...</td>
263      <td>-</td>
264    </tr>
265    <tr>
266      <th>1</th>
267      <td>US</td>
268      <td>11/03/2019</td>
269      <td>width: 99%;</td>
270      <td>So much fun</td>
271      <td>4.3.9</td>
272    </tr>
273    <tr>
274      <th>2</th>
275      <td>US</td>
276      <td>10/31/2019</td>
277      <td>width: 19%;</td>
278      <td>No woman</td>
279      <td>4.3.9</td>
280    </tr>
281    <tr>
282      <th>3</th>
283      <td>US</td>
284      <td>10/31/2019</td>
285      <td>width: 79%;</td>
286      <td>My camera is still not working</td>
287      <td>4.3.9</td>
288    </tr>
289    <tr>
290      <th>4</th>
291      <td>US</td>
292      <td>10/25/2019</td>
293      <td>width: 19%;</td>
294      <td>Cam broke with new iOS update just green lines</td>
295      <td>4.3.8</td>
296    </tr>
297    <tr>
298      <th>...</th>
299      <td>...</td>
300      <td>...</td>
301      <td>...</td>
302      <td>...</td>
303      <td>...</td>
304    </tr>
305    <tr>
306      <th>3179</th>
307      <td>US</td>
308      <td>
30807/19/2011</td>
309      <td>width: 99%;</td>
310      <td>Fun app glad I got it for free, would be aweso...</td>
311      <td>1.0</td>
312    </tr>
313    <tr>
314      <th>3180</th>
315      <td>US</td>
316      <td>07/19/2011</td>
317      <td>width: 39%;</td>
318      <td>Love this on iPad, but I'm trying to download ...</td>
319      <td>-</td>
320    </tr>
321    <tr>
322      <th>3181</th>
323      <td>US</td>
324      <td>07/18/2011</td>
325      <td>width: 59%;</td>
326      <td>Great but drops convo all tge time :(</td>
327      <td>-</td>
328    </tr>
329    <tr>
330      <th>3182</th>
331      <td>US</td>
332      <td>07/18/2011</td>
333      <td>width: 99%;</td>
334      <td>Works just like the service it connects to.</td>
335      <td>-</td>
336    </tr>
337    <tr>
338      <th>3183</th>
339      <td>US</td>
340      <td>07/16/2011</td>
341      <td>width: 19%;</td>
342      <td>This app is a waste of money. Connect randomly...</td>
343      <td>-</td>
344    </tr>
345  </tbody>
346</table>
347<p>3184 rows × 5 columns</p>
348</div>
349</div>
350
351</div>
352
353</div>
354</div>
355
356</div>
357<div class="cell border-box-sizing code_cell rendered">
358<div class="input">
359
360<div class="inner_cell">
361    <div class="input_area">
362<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># You&#39;ll change this filename for each app you&#39;re storing reviews for</span>
363<span class="n">df</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s2">&quot;data/chat-for-strangers.csv&quot;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
364</pre></div>
365
366    </div>
367</div>
368</div>
369
370</div>
371<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
372<div class="text_cell_render border-box-sizing rendered_html">
373<h2 id="Combine-and-add-columns">Combine and add columns<a class="anchor-link" href="#Combine-and-add-columns">#</a></h2><p>Once we've saved reviews for several different apps, we're ready to go. We'll combine them all into one single file and add a note about what app each review came from.</p>
374
375</div>
376</div>
377</div>
378<div class="cell border-box-sizing code_cell rendered">
379<div class="input">
380
381<div class="inner_cell">
382    <div class="input_area">
383<div class=" highlight hl-ipython3"><pre><span></span><span class="n">holla</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;data/holla.csv&#39;</span><span class="p">)</span>
384<span class="n">holla</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;holla&#39;</span>
385
386<span class="n">yubo</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;data/yubo.csv&#39;</span><span class="p">)</span>
387<span class="n">yubo</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;yubo&#39;</span>
388
389<span class="n">skout</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;data/skout.csv&#39;</span><span class="p">)</span>
390<span class="n">skout</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;skout&#39;</span>
391
392<span class="n">strangers</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="s1">&#39;data/chat-for-strangers.csv&#39;</span><span class="p">)</span>
393<span class="n">strangers</span><span class="p">[</span><span class="s1">&#39;source&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s1">&#39;chat-for-strangers&#39;</span>
394</pre></div>
395
396    </div>
397</div>
398</div>
399
400</div>
401<div class="cell border-box-sizing code_cell rendered">
402<div class="input">
403
404<div class="inner_cell">
405    <div class="input_area">
406<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">holla</span><span class="p">,</span> <span class="n">yubo</span><span class="p">,</span> <span class="n">skout</span><span class="p">,</span> <span class="n">strangers</span><span class="p">],</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
407<span class="n">df</span><span class="o">.</span><span class="n">shape</span>
408</pre></div>
409
410    </div>
411</div>
412</div>
413
414<div class="output_wrapper">
415<div class="output">
416
417<div class="output_area">
418
419
420
421<div class="output_text output_subarea output_execute_result">
422<pre>(56056, 6)</pre>
423</div>
424
425</div>
426
427</div>
428</div>
429
430</div>
431<div class="cell border-box-sizing code_cell rendered">
432<div class="input">
433
434<div class="inner_cell">
435    <div class="input_area">
436<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">source</span><span class="o">.</span><span class="n">value_counts</span><span class="p">()</span>
437</pre></div>
438
439    </div>
440</div>
441</div>
442
443<div class="output_wrapper">
444<div class="output">
445
446<div class="output_area">
447
448
449
450<div class="output_text output_subarea output_execute_result">
451<pre>skout                 37484
452holla                 10467
453yubo                   4921
454chat-for-strangers     3184
455Name: source, dtype: int64</pre>
456</div>
457
458</div>
459
460</div>
461</div>
462
463</div>
464<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
465<div class="text_cell_render border-box-sizing rendered_html">
466<p>We'll also add columns for racism, bullying, and unwanted sexual behavior. While we don't know 
466which reviews contain this content yet, we'll use these columns to mark it in Excel or Google Sheets later.</p>
467
468</div>
469</div>
470</div>
471<div class="cell border-box-sizing code_cell rendered">
472<div class="input">
473
474<div class="inner_cell">
475    <div class="input_area">
476<div class=" highlight hl-ipython3"><pre><span></span><span class="c1"># Using a machine learning algorithm to identify App Store reviews</span>
477<span class="c1"># containing reports of unwanted sexual content, racism and bullying...</span>
478<span class="n">df</span><span class="p">[</span><span class="s1">&#39;racism&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>
479<span class="n">df</span><span class="p">[</span><span class="s1">&#39;bullying&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>
480<span class="n">df</span><span class="p">[</span><span class="s1">&#39;sexual&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>
481
482<span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
483</pre></div>
484
485    </div>
486</div>
487</div>
488
489<div class="output_wrapper">
490<div class="output">
491
492<div class="output_area">
493
494
495<div class="output_html rendered_html output_subarea output_execute_result">
496<div>
497<style scoped>
498    .dataframe tbody tr th:only-of-type {
499        vertical-align: middle;
500    }
501
502    .dataframe tbody tr th {
503        vertical-align: top;
504    }
505
506    .dataframe thead th {
507        text-align: right;
508    }
509</style>
510<table border="1" class="dataframe">
511  <thead>
512    <tr style="text-align: right;">
513      <th></th>
514      <th>Country</th>
515      <th>Date</th>
516      <th>Rating</th>
517      <th>Review</th>
518      <th>Version</th>
519      <th>source</th>
520      <th>racism</th>
521      <th>bullying</th>
522      <th>sexual</th>
523    </tr>
524  </thead>
525  <tbody>
526    <tr>
527      <th>0</th>
528      <td>US</td>
529      <td>11/22/2019</td>
530      <td>width: 99%;</td>
531      <td>It’s a great app to meet new people and chat i...</td>
532      <td>4.4.5</td>
533      <td>holla</td>
534      <td>NaN</td>
535      <td>NaN</td>
536      <td>NaN</td>
537    </tr>
538    <tr>
539      <th>1</th>
540      <td>US</td>
541      <td>11/22/2019</td>
542      <td>width: 99%;</td>
543      <td>Holla is an excellent app, where I get to know...</td>
544      <td>4.4.5</td>
545      <td>holla</td>
546      <td>NaN</td>
547      <td>NaN</td>
548      <td>NaN</td>
549    </tr>
550    <tr>
551      <th>2</th>
552      <td>US</td>
553      <td>11/22/2019</td>
554      <td>width: 19%;</td>
555      <td>This app charges for everything now and is con...</td>
556      <td>-</td>
557      <td>holla</td>
558      <td>NaN</td>
559      <td>NaN</td>
560      <td>NaN</td>
561    </tr>
562    <tr>
563      <th>3</th>
564      <td>US</td>
565      <td>11/22/2019</td>
566      <td>width: 99%;</td>
567      <td>Free to use app, meet people around the world.</td>
568      <td>-</td>
569      <td>holla</td>
570      <td>NaN</td>
571      <td>NaN</td>
572      <td>NaN</td>
573    </tr>
574    <tr>
575      <th>4</th>
576      <td>US</td>
577      <td>11/21/2019</td>
578      <td>width: 99%;</td>
579      <td>I got this app and everything has been differe...</td>
580      <td>4.4.5</td>
581      <td>holla</td>
582      <td>NaN</td>
583      <td>NaN</td>
584      <td>NaN</td>
585    </tr>
586  </tbody>
587</table>
588</div>
589</div>
590
591</div>
592
593</div>
594</div>
595
596</div>
597<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
598<div class="text_cell_render border-box-sizing rendered_html">
599<h3 id="Clean-up-the-rating">Clean up the rating<a class="anchor-link" href="#Clean-up-the-rating">#</a></h3><p>We don't have ratings that are numeric! Let's convert the weird HTML star percentage to actual numbers.</p>
600
601</div>
602</div>
603</div>
604<div class="cell border-box-sizing code_cell rendered">
605<div class="input">
606
607<div class="inner_cell">
608    <div class="input_area">
609<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">Rating</span><span class="o">.</span><span class="n">value_counts</span><span class="p">()</span>
610</pre></div>
611
612    </div>
613</div>
614</div>
615
616<div class="output_wrapper">
617<div class="output">
618
619<div class="output_area">
620
621
622
623<div class="output_text output_subarea output_execute_result">
624<pre>width: 99%;    32761
625width: 19%;     8807
626width: 79%;     6418
627width: 59%;     4885
628width: 39%;     3185
629Name: Rating, dtype: int64</pre>
630</div>
631
632</div>
633
634</div>
635</div>
636
637</div>
638<div class="cell border-box-sizing code_cell rendered">
639<div class="input">
640
641<div class="inner_cell">
642    <div class="input_area">
643<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">Rating</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">Rating</span><span class="o">.</span><span class="n">replace</span><span class="p">({</span>
644    <span class="s1">&#39;width: 99%;&#39;</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span>
645    <span class="s1">&#39;width: 79%;&#39;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span>
646    <span class="s1">&#39;width: 59%;&#39;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
647    <span class="s1">&#39;width: 39%;&#39;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span>
648    <span class="s1">&#39;width: 19%;&#39;</span><span class="p">:</span> <span class="mi">1</span>
649<span class="p">})</span>
650<span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
651</pre></div>
652
653    </div>
654</div>
655</div>
656
657<div class="output_wrapper">
658<div class="output">
659
660<div class="output_area">
661
662
663<div class="output_html rendered_html output_subarea output_execute_result">
664<div>
665<style scoped>
666    .dataframe tbody tr th:only-of-type {
667        vertical-align: middle;
668    }
669
670    .dataframe tbody tr th {
671        vertical-align: top;
672    }
673
674    .dataframe thead th {
675        text-align: right;
676    }
677</style>
678<table border="1" class="dataframe">
679  <thead>
680    <tr style="text-align: right;">
681      <th></th>
682      <th>Country</th>
683      <th>Date</th>
684      <th>Rating</th>
685      <th>Review</th>
686      <th>Version</th>
687      <th>source</th>
688    </tr>
689  </thead>
690  <tbody>
691    <tr>
692      <th>0</th>
693      <td>US</td>
694      <td>11/22/2019</td>
695      <td>5</td>
696      <td>It’s a great app to meet new people and chat i...</td>
697      <td>4.4.5</td>
698      <td>holla</td>
699    </tr>
700    <tr>
701      <th>1</th>
702      <td>US</td>
703      <td>11/22/2019</td>
704      <td>5</td>
705      <td>Holla is an excellent app, where I get to know...</td>
706      <td>4.4.5</td>
707      <td>holla</td>
708    </tr>
709    <tr>
710      <th>2</th>
711      <td>US</td>
712      <td>11/22/2019</td>
713      <td>1</td>
714      <td>This app charges for everything now and is con...</td>
715      <td>-</td>
716      <td>holla</td>
717    </tr>
718    <tr>
719      <th>3</th>
720      <td>US</td>
721      <td>11/22/2019</td>
722      <td>5</td>
723      <td>Free to use app, meet people around the world.</td>
724      <td>-</td>
725      <td>holla</td>
726    </tr>
727    <tr>
728      <th>4</th>
729      <td>US</td>
730      <td>11/21/2019</td>
731      <td>5</td>
732      <td>I got this app and everything has been differe...</td>
733      <td>4.4.5</td>
734      <td>holla</td>
735    </tr>
736  </tbody>
737</table>
738</div>
739</div>
740
741</div>
742
743</div>
744</div>
745
746</div>
747<div class="cell border-box-sizing code_cell rendered">
748<div class="input">
749
750<div class="inner_cell">
751    <div class="input_area">
752<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">Rating</span><span class="o">.</span><span class="n">value_counts</span><span class="p">()</span>
753</pre></div>
754
755    </div>
756</div>
757</div>
758
759<div class="output_wrapper">
760<div class="output">
761
762<div class="output_area">
763
764
765
766<div class="output_text output_subarea output_execute_result">
767<pre>5    32761
7681     8807
7694     6418
7703     4885
7712     3185
772Name: Rating, dtype: int64</pre>
773</div>
774
775</div>
776
777</div>
778</div>
779
780</div>
781<div class="cell border-box-sizing code_cell rendered">
782<div class="input">
783
784<div class="inner_cell">
785    <div class="input_area">
786<div class=" highlight hl-ipython3"><pre><span></span><span class="n">df</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="s2">&quot;data/reviews.csv&quot;</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
787</pre></div>
788
789    </div>
790</div>
791</div>
792
793</div>
794<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
795<div class="text_cell_render border-box-sizing rendered_html">
796<h2 id="Review">Review<a class="anchor-link" href="#Review">#</a></h2><p>Instead of asking Apple or finding a secret API like the Washington Post, we used an <strong>app marketing site</strong> to find App Store reviews of the apps we were interested in. They didn't have a download button, though, so we wrote a simple scraper to pull them down.</p>
797<p>After obtaining the reviews, cleaned them a bit and we combined them into one spreadsheet and added columns for racism, bullying, and unwanted sexual behavior that we'll <strong>fill in later manually</strong>.</p>
798
799</div>
800</div>
801</div>
802<div class="cell border-box-sizing text_cell rendered"><div class="inner_cell">
803<div class="text_cell_render border-box-sizing rendered_html">
804<h2 id="Discussion-topics">Discussion topics<a class="anchor-link" href="#Discussion-topics">#</a></h2><p>Is pulling data from a secondary source okay?</p>
805<p>How do we know that they list all available reviews on the site that we obtained the reviews from?</p>
806<p>Do we need all of the reviews, or could we have filtered them at this point to narrow our field down?</p>
807
808</div>
809</div>
810</div>
811<div class="cell border-box-sizing code_cell rendered">
812<div class="input">
813
814<div class="inner_cell">
815    <div class="input_area">
816<div class=" highlight hl-ipython3"><pre><span></span> 
817</pre></div>
818
819    </div>
820</div>
821</div>
822
823</div>
824 
825
826
827
828      <div class="columns next-prev-links">
829        <div class="column col-6 col-sm-12 prev-column"></div>
830        <div class="column col-6 col-sm-12 text-right next-column"></div>
831      </div>
832    </div>
833  </div>
834</div>
835
836<script>
837  [...document.querySelectorAll(".code_cell")].filter(cell => cell.innerText.trim().length === 0).forEach(cell => cell.remove())
838</script>
838
839
840        <div class="footer bg-secondary">
841  <div class="content">
842    <div class="columns">
843      <div class="column col-8 col-sm-12">
844        <p><strong>About the site</strong></p>
845        <p>Hi, I'm <a href="https://twitter.com/dangerscarf">Soma</a>, welcome to Data Science for Journalism a.k.a. investigate.ai!</p>
846        <p>There's been a lot of buzz about machine learning and "artificial intelligence" being used in stories over the past few years. It's mostly not that complicated - a little stats, a classifier here or there - but it's hard to know where to start without a little help.</p>
847        <p>If you know a little Python programming, hopefully this site can be that help! <a href="/about">Learn more about this project here.</a></p>
848        <p><strong>Our newsletter</strong></p>
849        <form action="https://littlecolumns.us12.list-manage.com/subscribe/post?u=ecdebf156be0b7e068fac7c25&amp;id=16537e7c90" method="post" role="form" target="_blank" novalidate>
850          <div class="input-group">
851            <input name="EMAIL" type="email" class="form-input" placeholder="Enter your Email" required="">
852            <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_ecdebf156be0b7e068fac7c25_16537e7c90" tabindex="-1" value=""></div>
853            <div class="input-group-append">
854              <button class="btn btn-primary input-group-btn" type="submit" aria-label="Email signup submit button">
855                <i class="fas fa-envelope"></i>
856                Sign up
857              </button>
858            </div>
859          </div>
860        </form>
861        
862      </div>
863      <div class="column col-4 col-sm-12">
864        <p><strong>Links</strong></p>
865        <ul>
866          <li><a href="mailto:[email protected]">[email protected]</a></li>
867          <li><a href="https://twitter.com/dangerscarf">@dangerscarf</a></li>
868          <li><a href="/privacy-policy/">Privacy policy</a></li>
869          <li><a href="/newsletter/">Newsletter</a></li>
870          <li>Images via <a href="https://icons8.com/">icons8</a></li>
871        </ul>
872        <p>Thanks to <a href="https://journalism.columbia.edu/">Columbia Journalism School</a>, the <a href="https://knightfoundation.org/">Knight Foundation</a>, and <a href="/about#thankyou">many others</a>.</p>
873      </div>
874    </div>
875  </div>
876</div>
877<script src="/js/scripts.js?v=e9d8cec8fc9221c9dd15f6e0314372b3" type="text/javascript"></script>
877
878      </div>
879      <div class="bg-dark content-sidebar">
880        <div class="sidebar-holder">
881  <div class="brand">
882    <a href="/">investigate.ai</a>
883    <br>
884    data science for everybody
885  </div>
886  
887  <div class="accordion-container">
888    <div class="accordion">
889      <div class="form-group sidebar-sticky">
890        <form method="GET" action="/search/" class="search-form">
891          <div class="input-group">
892            <input type="text" class="form-input input-sm" name="q" placeholder="Search investigate.ai">
893            <button class="btn btn-primary input-group-btn btn-sm">Search</button>
894          </div>
895          </form>
896      </div>
897    
898        
899          <h4 id="textanalysis" class="sidebar-sticky">
900            <a href="#textanalysis">Text analysis</a>
901          </h4>
902          <ol class="menu menu-nav">
903            
904              <li class="menu-item">
905                
906  <a href="/text-analysis/types-of-text-analysis/">        
907      Types of text analysis
908  </a>
909  <div class="toc"></div>
910
911
912
913              </li>
914            
915              <li class="menu-item">
916                
917
918
919  <input id="accordion-countingwords" type="checkbox" name="accordion-checkbox" hidden="">
920  <label class="accordion-header c-hand" for="accordion-countingwords">
921      <i class="icon icon-arrow-right mr-1"></i>
922      Counting words
923  </label>
924  <div class="accordion-body">
925      <ol class="menu menu-nav">
926      
927      <li class="menu-item">
928          
929  <a href="/text-analysis/counting-words-with-pythons-counter/">        
930      Simple word counting
931  </a>
932  <div class="toc"></div>
933
934
935
936      </li>
937      
938      <li class="menu-item">
939          
940  <a href="/text-analysis/counting-words-with-scikit-learns-countvectorizer/">        
941      Counting words across many documents
942  </a>
943  <div class="toc"></div>
944
945
946
947      </li>
948      
949      <li class="menu-item">
950          
951  <a href="/text-analysis/splitting-words-in-east-asian-languages/">        
952      Segmenting words in East Asian languages
953  </a>
954  <div class="toc"></div>
955
956
957
958      </li>
959      
960      <li class="menu-item">
961          
962  <a href="/caixin-museum-word-count/counting-words-in-chinese-museum-names/">        
963      Project: Caixin museums
964  </a>
965  <div class="toc"></div>
966
967
968
969      </li>
970      
971      <li class="menu-item">
972          
973  <a href="/text-analysis/how-to-make-scikit-learn-natural-language-processing-work-with-japanese-chinese/">        
974      Using scikit-learn vectorizers with East Asian languages
975  </a>
976  <div class="toc"></div>
977
978
979
980      </li>
981      
982    </ol>
983  </div>
984
985              </li>
986            
987              <li class="menu-item">
988                
989
990
991  <input id="accordion-advancedwordanalysis" type="checkbox" name="accordion-checkbox" hidden="">
992  <label class="accordion-header c-hand" for="accordion-advancedwordanalysis">
993      <i class="icon icon-arrow-right mr-1"></i>
994      Advanced word analysis
995  </label>
996  <div class="accordion-body">
997      <ol class="menu menu-nav">
998      
999      <li class="menu-item">
1000          
1001  <a href="/text-analysis/a-simple-explanation-of-tf-idf/">        
1002      Upgraded word counts with TF-IDF
1003  </a>
1004  <div class="toc"></div>
1005
1006
1007
1008      </li>
1009      
1010      <li class="menu-item">
1011          
1012  <a href="/text-analysis/explaining-n-grams-in-natural-language-processing/">        
1013      Multi-word phrases and n-grams
1014  </a>
1015  <div class="toc"></div>
1016
1017
1018
1019      </li>
1020      
1021      <li class="menu-item">
1022          
1023  <a href="/text-analysis/stemming-and-lemmatization/">        
1024      Standardizing text with stemming and lemmatization
1025  </a>
1026  <div class="toc"></div>
1027
1028
1029
1030      </li>
1031      
1032      <li class="menu-item">
1033          
1034  <a href="/text-analysis/using-tf-idf-with-chinese/">        
1035      Using TF-IDF with Chinese text
1036  </a>
1037  <div class="toc"></div>
1038
1039
1040
1041      </li>
1042      
1043    </ol>
1044  </div>
1045
1046              </li>
1047            
1048              <li class="menu-item">
1049                
1050
1051
1052  <input id="accordion-sentimentanalysis" type="checkbox" name="accordion-checkbox" hidden="">
1053  <label class="accordion-header c-hand" for="accordion-sentimentanalysis">
1054      <i class="icon icon-arrow-right mr-1"></i>
1055      Sentiment analysis
1056  </label>
1057  <div class="accordion-body">
1058      <ol class="menu menu-nav">
1059      
1060      <li class="menu-item">
1061          
1062  <a href="/investigating-sentiment-analysis/comparing-sentiment-analysis-tools/">        
1063      Comparing sentiment analysis tools
1064  </a>
1065  <div class="toc"></div>
1066
1067
1068
1069      </li>
1070      
1071      <li class="menu-item">
1072          
1073  <a href="/investigating-sentiment-analysis/designing-your-own-sentiment-analysis-tool/">        
1074      Design your own sentiment analyzer
1075  </a>
1076  <div class="toc"></div>
1077
1078
1079
1080      </li>
1081      
1082      <li class="menu-item">
1083          
1084  <a href="/investigating-sentiment-analysis/more-data-to-train-our-sentiment-analysis-tool/">        
1085      Improving your tool
1086  </a>
1087  <div class="toc"></div>
1088
1089
1090
1091      </li>
1092      
1093      <li class="menu-item">
1094          
1095  <a href="/upshot-trump-emolex/nrc-emotional-lexicon/">        
1096      NRC Emotional Lexicon
1097  </a>
1098  <div class="toc"></div>
1099
1100
1101
1102      </li>
1103      
1104      <li class="menu-item">
1105          
1106  <a href="/upshot-trump-emolex/trump-vs-state-of-the-union-addresses/">        
1107      Project: UpShot State of the Union
1108  </a>
1109  <div class="toc"></div>
1110
1111
1112
1113      </li>
1114      
1115      <li class="menu-item">
1116          
1117  <a href="/nyt-trump-tweets/">        
1118      Project: NYT Trump tweets
1119  </a>
1120  <div class="toc"></div>
1121
1122
1123
1124      </li>
1125      
1126    </ol>
1127  </div>
1128
1129              </li>
1130            
1131              <li class="menu-item">
1132                
1133
1134
1135  <input id="accordion-documentstotext" type="checkbox" name="accordion-checkbox" hidden="">
1136  <label class="accordion-header c-hand" for="accordion-documentstotext">
1137      <i class="icon icon-arrow-right mr-1"></i>
1138      Documents to text
1139  </label>
1140  <div class="accordion-body">
1141      <ol class="menu menu-nav">
1142      
1143      <li class="menu-item">
1144          
1145  <a href="/text-analysis/processing-documents-with-apache-tika/">        
1146      Converting documents to text (English)
1147  </a>
1148  <div class="toc"></div>
1149
1150
1151
1152      </li>
1153      
1154      <li class="menu-item">
1155          
1156  <a href="/text-analysis/processing-documents-with-apache-tika-greek/">        
1157      Converting documents to text (non-English)
1158  </a>
1159  <div class="toc"></div>
1160
1161
1162
1163      </li>
1164      
1165    </ol>
1166  </div>
1167
1168              </li>
1169            
1170              <li class="menu-item">
1171                
1172
1173
1174  <input id="accordion-conceptspeopleandplaces" type="checkbox" name="accordion-checkbox" hidden="">
1175  <label class="accordion-header c-hand" for="accordion-conceptspeopleandplaces">
1176      <i class="icon icon-arrow-right mr-1"></i>
1177      Concepts, people and places
1178  </label>
1179  <div class="accordion-body">
1180      <ol class="menu menu-nav">
1181      
1182      <li class="menu-item">
1183          
1184  <a href="/text-analysis/introduction-to-topic-modeling/">        
1185      Extracting topics from documents
1186  </a>
1187  <div class="toc"></div>
1188
1189
1190
1191      </li>
1192      
1193      <li class="menu-item">
1194          
1195  <a href="/text-analysis/choosing-the-right-number-of-topics-for-a-scikit-learn-topic-model/">        
1196      Choosing the right number of topics
1197  </a>
1198  <div class="toc"></div>
1199
1200
1201
1202      </li>
1203      
1204      <li class="menu-item">
1205          
1206  <a href="/text-analysis/topic-models-with-gensim/">        
1207      Topic models with Gensim
1208  </a>
1209  <div class="toc"></div>
1210
1211
1212
1213      </li>
1214      
1215      <li class="menu-item">
1216          
1217  <a href="/text-analysis/topic-modeling-and-clustering/">        
1218      Topic models vs clustering
1219  </a>
1220  <div class="toc"></div>
1221
1222
1223
1224      </li>
1225      
1226      <li class="menu-item">
1227          
1228  <a href="/text-analysis/named-entity-recognition/">        
1229      Entity recognition
1230  </a>
1231  <div class="toc"></div>
1232
1233
1234
1235      </li>
1236      
1237      <li class="menu-item">
1238          
1239  <a href="/text-analysis/word-embeddings/">        
1240      Intro to word embeddings
1241  </a>
1242  <div class="toc"></div>
1243
1244
1245
1246      </li>
1247      
1248      <li class="menu-item">
1249          
1250  <a href="/text-analysis/document-similarity-using-word-embeddings/">        
1251      Conceptual document similarity
1252  </a>
1253  <div class="toc"></div>
1254
1255
1256
1257      </li>
1258      
1259      <li class="menu-item">
1260          
1261  <a href="/text-analysis/comparing-documents-in-different-languages/">        
1262      Comparing documents in different languages
1263  </a>
1264  <div class="toc"></div>
1265
1266
1267
1268      </li>
1269      
1270    </ol>
1271  </div>
1272
1273              </li>
1274            
1275          </ol>
1276        
1277          <h4 id="puttingthingsincategoriesautomatically" class="sidebar-sticky">
1278            <a href="#puttingthingsincategoriesautomatically">Putting things in categories automatically</a>
1279          </h4>
1280          <ol class="menu menu-nav">
1281            
1282              <li class="menu-item">
1283                
1284  <a href="/classification/intro-to-classification/">        
1285      Introduction to Classification
1286  </a>
1287  <div class="toc"></div>
1288
1289
1290
1291              </li>
1292            
1293              <li class="menu-item">
1294                
1295
1296
1297  <input id="accordion-techniques" type="checkbox" name="accordion-checkbox" hidden="">
1298  <label class="accordion-header c-hand" for="accordion-techniques">
1299      <i class="icon icon-arrow-right mr-1"></i>
1300      Techniques
1301  </label>
1302  <div class="accordion-body">
1303      <ol class="menu menu-nav">
1304      
1305      <li class="menu-item">
1306          
1307  <a href="/classification/evaluating-classifiers/">        
1308      Evaluating classifiers
1309  </a>
1310  <div class="toc"></div>
1311
1312
1313
1314      </li>
1315      
1316      <li class="menu-item">
1317          
1318  <a href="/classification/scikit-learn-and-categorical-features/">        
1319      Categorical features
1320  </a>
1321  <div class="toc"></div>
1322
1323
1324
1325      </li>
1326      
1327      <li class="menu-item">
1328          
1329  <a href="/classification/using-classification-algorithms-with-text/">        
1330      Classifiers with text
1331  </a>
1332  <div class="toc"></div>
1333
1334
1335
1336      </li>
1337      
1338      <li class="menu-item">
1339          
1340  <a href="/classification/correcting-for-imbalanced-datasets/">        
1341      Correcting for imbalanced datasets
1342  </a>
1343  <div class="toc"></div>
1344
1345
1346
1347      </li>
1348      
1349    </ol>
1350  </div>
1351
1352              </li>
1353            
1354              <li class="menu-item">
1355                
1356
1357
1358  <input id="accordion-projects" type="checkbox" name="accordion-checkbox" hidden="">
1359  <label class="accordion-header c-hand" for="accordion-projects">
1360      <i class="icon icon-arrow-right mr-1"></i>
1361      Projects
1362  </label>
1363  <div class="accordion-body">
1364      <ol class="menu menu-nav">
1365      
1366      <li class="menu-item">
1367          
1368  <a href="/buzzfeed-spy-planes/buzzfeed-surveillance-planes-random-forests/">        
1369      BuzzFeed: Spy planes
1370  </a>
1371  <div class="toc"></div>
1372
1373
1374
1375      </li>
1376      
1377      <li class="menu-item">
1378          
1379  <a href="/wapo-app-reviews/predict-reviews/">        
1380      WaPo chat: App reviews
1381  </a>
1382  <div class="toc"></div>
1383
1384
1385
1386      </li>
1387      
1388      <li class="menu-item">
1389          
1390  <a href="/nyt-takata-airbags/nyt-takata-completed/">        
1391      NYT: Faulty airbag search
1392  </a>
1393  <div class="toc"></div>
1394
1395
1396
1397      </li>
1398      
1399      <li class="menu-item">
1400          
1401  <a href="/latimes-crime-classification/using-a-classifier-to-find-misclassified-crimes/">        
1402      LA Times: crime classifier
1403  </a>
1404  <div class="toc"></div>
1405
1406
1407
1408      </li>
1409      
1410    </ol>
1411  </div>
1412
1413              </li>
1414            
1415          </ol>
1416        
1417          <h4 id="howxaffectsy" class="sidebar-sticky">
1418            <a href="#howxaffectsy">How X affects Y</a>
1419          </h4>
1420          <ol class="menu menu-nav">
1421            
1422              <li class="menu-item">
1423                
1424  <a href="/regression/what-is-regression/">        
1425      Finding relationships with regression
1426  </a>
1427  <div class="toc"></div>
1428
1429
1430
1431              </li>
1432            
1433              <li class="menu-item">
1434                
1435
1436
1437  <input id="accordion-linearregression" type="checkbox" name="accordion-checkbox" hidden="">
1438  <label class="accordion-header c-hand" for="accordion-linearregression">
1439      <i class="icon icon-arrow-right mr-1"></i>
1440      Linear Regression
1441  </label>
1442  <div class="accordion-body">
1443      <ol class="menu menu-nav">
1444      
1445      <li class="menu-item">
1446          
1447  <a href="/regression/linear-regression-quickstart/">        
1448      Linear regression (Quickstart)
1449  </a>
1450  <div class="toc"></div>
1451
1452
1453
1454      </li>
1455      
1456      <li class="menu-item">
1457          
1458  <a href="/regression/linear-regression/">        
1459      Linear regression for humans
1460  </a>
1461  <div class="toc"></div>
1462
1463
1464
1465      </li>
1466      
1467      <li class="menu-item">
1468          
1469  <a href="/regression/linear-regression-part-two/">        
1470      Putting regression to use
1471  </a>
1472  <div class="toc"></div>
1473
1474
1475
1476      </li>
1477      
1478      <li class="menu-item">
1479          
1480  <a href="/regression/linear-regression-evaluation/">        
1481      Evaluating regressions
1482  </a>
1483  <div class="toc"></div>
1484
1485
1486
1487      </li>
1488      
1489      <li class="menu-item">
1490          
1491  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-formulas/">        
1492      Associated Press: Life expectancy and unemployment
1493  </a>
1494  <div class="toc"></div>
1495
1496
1497
1498      </li>
1499      
1500    </ol>
1501  </div>
1502
1503              </li>
1504            
1505              <li class="menu-item">
1506                
1507
1508
1509  <input id="accordion-logisticregression" type="checkbox" name="accordion-checkbox" hidden="">
1510  <label class="accordion-header c-hand" for="accordion-logisticregression">
1511      <i class="icon icon-arrow-right mr-1"></i>
1512      Logistic Regression
1513  </label>
1514  <div class="accordion-body">
1515      <ol class="menu menu-nav">
1516      
1517      <li class="menu-item">
1518          
1519  <a href="/regression/logistic-regression-quickstart/">        
1520      Logistic regression (Quickstart)
1521  </a>
1522  <div class="toc"></div>
1523
1524
1525
1526      </li>
1527      
1528      <li class="menu-item">
1529          
1530  <a href="/regression/logistic-regression/">        
1531      Logistic regression for humans
1532  </a>
1533  <div class="toc"></div>
1534
1535
1536
1537      </li>
1538      
1539      <li class="menu-item">
1540          
1541  <a href="/regression/logistic-regression-part-two/">        
1542      More complex logistic regressions
1543  </a>
1544  <div class="toc"></div>
1545
1546
1547
1548      </li>
1549      
1550      <li class="menu-item">
1551          
1552  <a href="/regression/evaluating-logistic-regressions/">        
1553      Evaluating logistic regressions
1554  </a>
1555  <div class="toc"></div>
1556
1557
1558
1559      </li>
1560      
1561      <li class="menu-item">
1562          
1563  <a href="/boston-globe-tickets/boston-globe-ticketing-regression/">        
1564      Boston Globe: Speeding tickets
1565  </a>
1566  <div class="toc"></div>
1567
1568
1569
1570      </li>
1571      
1572      <li class="menu-item">
1573          
1574  <a href="/apm-reports-jury-bias/in-the-dark-alternative-formula-methods/">        
1575      APM Reports: Jury selection
1576  </a>
1577  <div class="toc"></div>
1578
1579
1580
1581      </li>
1582      
1583    </ol>
1584  </div>
1585
1586              </li>
1587            
1588          </ol>
1589        
1590          <h4 id="pythondatasciencereference" class="sidebar-sticky">
1591            <a href="#pythondatasciencereference">Python data science reference</a>
1592          </h4>
1593          <ol class="menu menu-nav">
1594            
1595              <li class="menu-item">
1596                
1597  <a href="/reference/">        
1598      Introduction
1599  </a>
1600  <div class="toc"></div>
1601
1602
1603
1604              </li>
1605            
1606              <li class="menu-item">
1607                
1608  <a href="/reference/vectorizing/">        
1609      Vectorizing
1610  </a>
1611  <div class="toc"></div>
1612
1613
1614
1615              </li>
1616            
1617              <li class="menu-item">
1618                
1619  <a href="/reference/text-analysis/">        
1620      Text Analysis
1621  </a>
1622  <div class="toc"></div>
1623
1624
1625
1626              </li>
1627            
1628              <li class="menu-item">
1629                
1630  <a href="/reference/regression/">        
1631      Regression
1632  </a>
1633  <div class="toc"></div>
1634
1635
1636
1637              </li>
1638            
1639              <li class="menu-item">
1640                
1641  <a href="/reference/classification/">        
1642      Classification
1643  </a>
1644  <div class="toc"></div>
1645
1646
1647
1648              </li>
1649            
1650          </ol>
1651        
1652          <h4 id="allprojects" class="sidebar-sticky">
1653            <a href="#allprojects">All Projects</a>
1654          </h4>
1655          <ol class="menu menu-nav">
1656            
1657              <li class="menu-item">
1658                
1659  <a href="/projects/">        
1660      Project Summaries
1661  </a>
1662  <div class="toc"></div>
1663
1664
1665
1666              </li>
1667            
1668              <li class="menu-item">
1669                
1670
1671
1672  <input id="accordion-strongnytstrongtakataairbagsearch" type="checkbox" name="accordion-checkbox" hidden="">
1673  <label class="accordion-header c-hand" for="accordion-strongnytstrongtakataairbagsearch">
1674      <i class="icon icon-arrow-right mr-1"></i>
1675      <strong>NYT:</strong>  Takata airbag search
1676  </label>
1677  <div class="accordion-body">
1678      <ol class="menu menu-nav">
1679      
1680      <li class="menu-item">
1681          
1682  <a href="/nyt-takata-airbags/">        
1683      Summary
1684  </a>
1685  <div class="toc"></div>
1686
1687
1688
1689      </li>
1690      
1691      <li class="menu-item">
1692          
1693  <a href="/nyt-takata-airbags/airbag-classifier-search-binary/">        
1694      A simplistic reproduction of the NYT's research using logistic regression
1695  </a>
1696  <div class="toc"></div>
1697
1698
1699
1700      </li>
1701      
1702      <li class="menu-item">
1703          
1704  <a href="/nyt-takata-airbags/airbag-classifier-search-decision-tree/">        
1705      A decision-tree reproduction of the NYT's research
1706  </a>
1707  <div class="toc"></div>
1708
1709
1710
1711      </li>
1712      
1713      <li class="menu-item">
1714          
1715  <a href="/nyt-takata-airbags/airbag-classifier-search-countvectorizer/">        
1716      Combining a text vectorizer and a classifier to track down suspicious complaints
1717  </a>
1718  <div class="toc"></div>
1719
1720
1721
1722      </li>
1723      
1724    </ol>
1725  </div>
1726
1727              </li>
1728            
1729              <li class="menu-item">
1730                
1731
1732
1733  <input id="accordion-stronglatimesstrongcrimeclassification" type="checkbox" name="accordion-checkbox" hidden="">
1734  <label class="accordion-header c-hand" for="accordion-stronglatimesstrongcrimeclassification">
1735      <i class="icon icon-arrow-right mr-1"></i>
1736      <strong>LA Times:</strong>  Crime classification
1737  </label>
1738  <div class="accordion-body">
1739      <ol class="menu menu-nav">
1740      
1741      <li class="menu-item">
1742          
1743  <a href="/latimes-crime-classification/">        
1744      Summary
1745  </a>
1746  <div class="toc"></div>
1747
1748
1749
1750      </li>
1751      
1752      <li class="menu-item">
1753          
1754  <a href="/latimes-crime-classification/using-a-classifier-to-find-misclassified-crimes/">        
1755      Predicting downgraded assaults with machine learning
1756  </a>
1757  <div class="toc"></div>
1758
1759
1760
1761      </li>
1762      
1763      <li class="menu-item">
1764          
1765  <a href="/latimes-crime-classification/inspecting-classifications/">        
1766      Taking a closer look at our classifier and its misclassifications
1767  </a>
1768  <div class="toc"></div>
1769
1770
1771
1772      </li>
1773      
1774      <li class="menu-item">
1775          
1776  <a href="/latimes-crime-classification/trying-out-different-classifiers/">        
1777      Trying out and combining different classifiers
1778  </a>
1779  <div class="toc"></div>
1780
1781
1782
1783      </li>
1784      
1785    </ol>
1786  </div>
1787
1788              </li>
1789            
1790              <li class="menu-item">
1791                
1792
1793
1794  <input id="accordion-strongcaixinstrongmuseumnames" type="checkbox" name="accordion-checkbox" hidden="">
1795  <label class="accordion-header c-hand" for="accordion-strongcaixinstrongmuseumnames">
1796      <i class="icon icon-arrow-right mr-1"></i>
1797      <strong>Caixin:</strong>  Museum names
1798  </label>
1799  <div class="accordion-body">
1800      <ol class="menu menu-nav">
1801      
1802      <li class="menu-item">
1803          
1804  <a href="/caixin-museum-word-count/">        
1805      Summary
1806  </a>
1807  <div class="toc"></div>
1808
1809
1810
1811      </li>
1812      
1813      <li class="menu-item">
1814          
1815  <a href="/caixin-museum-word-count/chinese-museum-dataset-cleanup/">        
1816      Chinese museum dataset cleanup
1817  </a>
1818  <div class="toc"></div>
1819
1820
1821
1822      </li>
1823      
1824      <li class="menu-item">
1825          
1826  <a href="/caixin-museum-word-count/chinese-museums-per-capita-analysis/">        
1827      Chinese museums per capita analysis
1828  </a>
1829  <div class="toc"></div>
1830
1831
1832
1833      </li>
1834      
1835      <li class="menu-item">
1836          
1837  <a href="/caixin-museum-word-count/counting-words-in-chinese-museum-names/">        
1838      Counting words in Chinese museum names
1839  </a>
1840  <div class="toc"></div>
1841
1842
1843
1844      </li>
1845      
1846    </ol>
1847  </div>
1848
1849              </li>
1850            
1851              <li class="menu-item">
1852                
1853
1854
1855  <input id="accordion-strongwapostrongrandomchatappsafety" type="checkbox" name="accordion-checkbox" hidden="">
1856  <label class="accordion-header c-hand" for="accordion-strongwapostrongrandomchatappsafety">
1857      <i class="icon icon-arrow-right mr-1"></i>
1858      <strong>WaPo:</strong>  Random chat app safety
1859  </label>
1860  <div class="accordion-body">
1861      <ol class="menu menu-nav">
1862      
1863      <li class="menu-item">
1864          
1865  <a href="/wapo-app-reviews/">        
1866      Summary
1867  </a>
1868  <div class="toc"></div>
1869
1870
1871
1872      </li>
1873      
1874      <li class="menu-item">
1875          
1876  <a href="/wapo-app-reviews/scrape-app-store-reviews/">        
1877      Scrape and combine app store reviews
1878  </a>
1879  <div class="toc"></div>
1880
1881
1882
1883      </li>
1884      
1885      <li class="menu-item">
1886          
1887  <a href="/wapo-app-reviews/predict-reviews/">        
1888      Build a classifier to detect reviews about bad behavior
1889  </a>
1890  <div class="toc"></div>
1891
1892
1893
1894      </li>
1895      
1896    </ol>
1897  </div>
1898
1899              </li>
1900            
1901              <li class="menu-item">
1902                
1903  <a href="/ajc-doctors-abuse/">        
1904      <strong>AJC:</strong>  Doctors and sex abuse
1905  </a>
1906  <div class="toc"></div>
1907
1908
1909
1910              </li>
1911            
1912              <li class="menu-item">
1913                
1914
1915
1916  <input id="accordion-strongtheupshotstrongtrumpspeeches" type="checkbox" name="accordion-checkbox" hidden="">
1917  <label class="accordion-header c-hand" for="accordion-strongtheupshotstrongtrumpspeeches">
1918      <i class="icon icon-arrow-right mr-1"></i>
1919      <strong>The UpShot:</strong>  Trump speeches
1920  </label>
1921  <div class="accordion-body">
1922      <ol class="menu menu-nav">
1923      
1924      <li class="menu-item">
1925          
1926  <a href="/upshot-trump-emolex/">        
1927      Summary
1928  </a>
1929  <div class="toc"></div>
1930
1931
1932
1933      </li>
1934      
1935      <li class="menu-item">
1936          
1937  <a href="/upshot-trump-emolex/nrc-emotional-lexicon/">        
1938      An introduction to the NRC Emotional Lexicon
1939  </a>
1940  <div class="toc"></div>
1941
1942
1943
1944      </li>
1945      
1946      <li class="menu-item">
1947          
1948  <a href="/upshot-trump-emolex/trump-vs-state-of-the-union-addresses/">        
1949      Reproducing The UpShot's Trump State of the Union visualization
1950  </a>
1951  <div class="toc"></div>
1952
1953
1954
1955      </li>
1956      
1957    </ol>
1958  </div>
1959
1960              </li>
1961            
1962              <li class="menu-item">
1963                
1964
1965
1966  <input id="accordion-strongusatodaystrongmodellegislation" type="checkbox" name="accordion-checkbox" hidden="">
1967  <label class="accordion-header c-hand" for="accordion-strongusatodaystrongmodellegislation">
1968      <i class="icon icon-arrow-right mr-1"></i>
1969      <strong>USA Today:</strong>  Model legislation
1970  </label>
1971  <div class="accordion-body">
1972      <ol class="menu menu-nav">
1973      
1974      <li class="menu-item">
1975          
1976  <a href="/azcentral-text-reuse-model-legislation/">        
1977      Summary
1978  </a>
1979  <div class="toc"></div>
1980
1981
1982
1983      </li>
1984      
1985      <li class="menu-item">
1986          
1987  <a href="/azcentral-text-reuse-model-legislation/01-downloading-one-million-pieces-of-legislation-from-legiscan/">        
1988      Downloading one million pieces of legislation from LegiScan
1989  </a>
1990  <div class="toc"></div>
1991
1992
1993
1994      </li>
1995      
1996      <li class="menu-item">
1997          
1998  <a href="/azcentral-text-reuse-model-legislation/02-taking-a-mill
1998ion-pieces-of-legislation-from-a-csv-and-inserting-them-into-postgres/">        
1999      Taking a million pieces of legislation from a CSV and inserting them into Postgres
2000  </a>
2001  <div class="toc"></div>
2002
2003
2004
2005      </li>
2006      
2007      <li class="menu-item">
2008          
2009  <a href="/azcentral-text-reuse-model-legislation/03-download-word-pdf-and-html-content-and-process-it-into-text-with-tika/">        
2010      Download Word, PDF and HTML content and process it into text with Tika
2011  </a>
2012  <div class="toc"></div>
2013
2014
2015
2016      </li>
2017      
2018      <li class="menu-item">
2019          
2020  <a href="/azcentral-text-reuse-model-legislation/04-import-content-into-solr-for-advanced-text-searching/">        
2021      Import content into Solr for advanced text searching
2022  </a>
2023  <div class="toc"></div>
2024
2025
2026
2027      </li>
2028      
2029      <li class="menu-item">
2030          
2031  <a href="/azcentral-text-reuse-model-legislation/05-checking-for-legislative-text-reuse-using-python-solr-and-ngrams/">        
2032      Checking for legislative text reuse using Python, Solr, and ngrams
2033  </a>
2034  <div class="toc"></div>
2035
2036
2037
2038      </li>
2039      
2040      <li class="menu-item">
2041          
2042  <a href="/azcentral-text-reuse-model-legislation/05-checking-for-legislative-text-reuse-using-python-solr-and-simple-text-search/">        
2043      Checking for legislative text reuse using Python, Solr, and simple text search
2044  </a>
2045  <div class="toc"></div>
2046
2047
2048
2049      </li>
2050      
2051      <li class="menu-item">
2052          
2053  <a href="/azcentral-text-reuse-model-legislation/06-search-for-model-legislation-in-over-one-million-bills-using-postgres-and-solr/">        
2054      Search for model legislation in over one million bills using Postgres and Solr
2055  </a>
2056  <div class="toc"></div>
2057
2058
2059
2060      </li>
2061      
2062      <li class="menu-item">
2063          
2064  <a href="/azcentral-text-reuse-model-legislation/using-topic-modeling-to-categorize-legislation/">        
2065      Using topic modeling to categorize legislation
2066  </a>
2067  <div class="toc"></div>
2068
2069
2070
2071      </li>
2072      
2073    </ol>
2074  </div>
2075
2076              </li>
2077            
2078              <li class="menu-item">
2079                
2080  <a href="/fcc-comments/">        
2081      FCC comment bots
2082  </a>
2083  <div class="toc"></div>
2084
2085
2086
2087              </li>
2088            
2089              <li class="menu-item">
2090                
2091
2092
2093  <input id="accordion-strongbloombergstrongdemocraticcandidatetweets" type="checkbox" name="accordion-checkbox" hidden="">
2094  <label class="accordion-header c-hand" for="accordion-strongbloombergstrongdemocraticcandidatetweets">
2095      <i class="icon icon-arrow-right mr-1"></i>
2096      <strong>Bloomberg:</strong>  Democratic Candidate Tweets
2097  </label>
2098  <div class="accordion-body">
2099      <ol class="menu menu-nav">
2100      
2101      <li class="menu-item">
2102          
2103  <a href="/bloomberg-tweet-topics/">        
2104      Summary
2105  </a>
2106  <div class="toc"></div>
2107
2108
2109
2110      </li>
2111      
2112      <li class="menu-item">
2113          
2114  <a href="/bloomberg-tweet-topics/scrape-tweets-from-presidential-primary-candidates/">        
2115      Downloading all 2019 tweets from Democratic presidential candidates
2116  </a>
2117  <div class="toc"></div>
2118
2119
2120
2121      </li>
2122      
2123      <li class="menu-item">
2124          
2125  <a href="/bloomberg-tweet-topics/topic-modeling-for-tweets/">        
2126      Using topic modeling to analyze presidential candidate tweets
2127  </a>
2128  <div class="toc"></div>
2129
2130
2131
2132      </li>
2133      
2134      <li class="menu-item">
2135          
2136  <a href="/bloomberg-tweet-topics/assigning-categories-to-text-using-keyword-matching/">        
2137      Assigning categories to tweets using keyword matching
2138  </a>
2139  <div class="toc"></div>
2140
2141
2142
2143      </li>
2144      
2145      <li class="menu-item">
2146          
2147  <a href="/bloomberg-tweet-topics/building-streamgraphs-from-candidate-tweets/">        
2148      Building streamgraphs from categorized and dated datasets
2149  </a>
2150  <div class="toc"></div>
2151
2152
2153
2154      </li>
2155      
2156    </ol>
2157  </div>
2158
2159              </li>
2160            
2161              <li class="menu-item">
2162                
2163  <a href="/nyt-trump-tweets/">        
2164      <strong>NYT:</strong>  Trump tweets
2165  </a>
2166  <div class="toc"></div>
2167
2168
2169
2170              </li>
2171            
2172              <li class="menu-item">
2173                
2174
2175
2176  <input id="accordion-strongapstronglifeexpectancy" type="checkbox" name="accordion-checkbox" hidden="">
2177  <label class="accordion-header c-hand" for="accordion-strongapstronglifeexpectancy">
2178      <i class="icon icon-arrow-right mr-1"></i>
2179      <strong>AP:</strong>  Life expectancy
2180  </label>
2181  <div class="accordion-body">
2182      <ol class="menu menu-nav">
2183      
2184      <li class="menu-item">
2185          
2186  <a href="/ap-regression-unemployment/">        
2187      Summary
2188  </a>
2189  <div class="toc"></div>
2190
2191
2192
2193      </li>
2194      
2195      <li class="menu-item">
2196          
2197  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-formulas/">        
2198      Simple logistic regression using statsmodels (formula version)
2199  </a>
2200  <div class="toc"></div>
2201
2202
2203
2204      </li>
2205      
2206      <li class="menu-item">
2207          
2208  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-dataframes/">        
2209      Simple logistic regression using statsmodels (dataframes version)
2210  </a>
2211  <div class="toc"></div>
2212
2213
2214
2215      </li>
2216      
2217    </ol>
2218  </div>
2219
2220              </li>
2221            
2222              <li class="menu-item">
2223                
2224  <a href="/fivethirtyeight-p-hacking/">        
2225      <strong>FiveThirtyEight:</strong>  P-values
2226  </a>
2227  <div class="toc"></div>
2228
2229
2230
2231              </li>
2232            
2233              <li class="menu-item">
2234                
2235
2236
2237  <input id="accordion-strongmilwaukeejournalsentinelstrongpotholes" type="checkbox" name="accordion-checkbox" hidden="">
2238  <label class="accordion-header c-hand" for="accordion-strongmilwaukeejournalsentinelstrongpotholes">
2239      <i class="icon icon-arrow-right mr-1"></i>
2240      <strong>Milwaukee Journal-Sentinel:</strong>  Potholes
2241  </label>
2242  <div class="accordion-body">
2243      <ol class="menu menu-nav">
2244      
2245      <li class="menu-item">
2246          
2247  <a href="/milwaukee-potholes/">        
2248      Summary
2249  </a>
2250  <div class="toc"></div>
2251
2252
2253
2254      </li>
2255      
2256      <li class="menu-item">
2257          
2258  <a href="/milwaukee-potholes/milwaukee-journal-sentinel-and-potholes-full-walkthrough/">        
2259      Pothole geographic analysis and linear regression, complete walkthrough
2260  </a>
2261  <div class="toc"></div>
2262
2263
2264
2265      </li>
2266      
2267      <li class="menu-item">
2268          
2269  <a href="/milwaukee-potholes/milwaukee-journal-sentinel-and-potholes-without-merging/">        
2270      Pothole demographics linear regression, no spatial analysis
2271  </a>
2272  <div class="toc"></div>
2273
2274
2275
2276      </li>
2277      
2278    </ol>
2279  </div>
2280
2281              </li>
2282            
2283              <li class="menu-item">
2284                
2285
2286
2287  <input id="accordion-strongdallasmorningnewsstrongcheatingschools" type="checkbox" name="accordion-checkbox" hidden="">
2288  <label class="accordion-header c-hand" for="accordion-strongdallasmorningnewsstrongcheatingschools">
2289      <i class="icon icon-arrow-right mr-1"></i>
2290      <strong>Dallas Morning News:</strong>  Cheating schools
2291  </label>
2292  <div class="accordion-body">
2293      <ol class="menu menu-nav">
2294      
2295      <li class="menu-item">
2296          
2297  <a href="/dmn-texas-school-cheating/">        
2298      Summary
2299  </a>
2300  <div class="toc"></div>
2301
2302
2303
2304      </li>
2305      
2306      <li class="menu-item">
2307          
2308  <a href="/dmn-texas-school-cheating/texas-school-cheating-finding-outliers-with-standard-deviation-and-regression/">        
2309      Finding outliers with standard deviation and regression
2310  </a>
2311  <div class="toc"></div>
2312
2313
2314
2315      </li>
2316      
2317      <li class="menu-item">
2318          
2319  <a href="/dmn-texas-school-cheating/texas-school-cheating-finding-outliers-with-regression-residuals/">        
2320      Finding outliers with regression residuals (short version)
2321  </a>
2322  <div class="toc"></div>
2323
2324
2325
2326      </li>
2327      
2328      <li class="menu-item">
2329          
2330  <a href="/dmn-texas-school-cheating/texas-school-cheating-graph-reproductions/">        
2331      Reproducing the graphics from The Dallas Morning News piece
2332  </a>
2333  <div class="toc"></div>
2334
2335
2336
2337      </li>
2338      
2339    </ol>
2340  </div>
2341
2342              </li>
2343            
2344              <li class="menu-item">
2345                
2346
2347
2348  <input id="accordion-strongtampabaytimesstrongfailurefactories" type="checkbox" name="accordion-checkbox" hidden="">
2349  <label class="accordion-header c-hand" for="accordion-strongtampabaytimesstrongfailurefactories">
2350      <i class="icon icon-arrow-right mr-1"></i>
2351      <strong>Tampa Bay Times:</strong>  Failure factories
2352  </label>
2353  <div class="accordion-body">
2354      <ol class="menu menu-nav">
2355      
2356      <li class="menu-item">
2357          
2358  <a href="/tampa-bay-times-schools/">        
2359      Summary
2360  </a>
2361  <div class="toc"></div>
2362
2363
2364
2365      </li>
2366      
2367      <li class="menu-item">
2368          
2369  <a href="/tampa-bay-times-schools/linear-regression-on-florida-schools/">        
2370      Linear regression on Florida schools, complete walkthrough
2371  </a>
2372  <div class="toc"></div>
2373
2374
2375
2376      </li>
2377      
2378      <li class="menu-item">
2379          
2380  <a href="/tampa-bay-times-schools/linear-regression-on-florida-schools-no-cleaning/">        
2381      Linear regression on Florida schools, no cleaning
2382  </a>
2383  <div class="toc"></div>
2384
2385
2386
2387      </li>
2388      
2389    </ol>
2390  </div>
2391
2392              </li>
2393            
2394              <li class="menu-item">
2395                
2396
2397
2398  <input id="accordion-caraccidentsandcarweight" type="checkbox" name="accordion-checkbox" hidden="">
2399  <label class="accordion-header c-hand" for="accordion-caraccidentsandcarweight">
2400      <i class="icon icon-arrow-right mr-1"></i>
2401      Car accidents and car weight
2402  </label>
2403  <div class="accordion-body">
2404      <ol class="menu menu-nav">
2405      
2406      <li class="menu-item">
2407          
2408  <a href="/car-crashes-weight-regression/">        
2409      Summary
2410  </a>
2411  <div class="toc"></div>
2412
2413
2414
2415      </li>
2416      
2417      <li class="menu-item">
2418          
2419  <a href="/car-crashes-weight-regression/car-crashes-feature-selection-and-engineering/">        
2420      Feature selection and engineering
2421  </a>
2422  <div class="toc"></div>
2423
2424
2425
2426      </li>
2427      
2428      <li class="menu-item">
2429          
2430  <a href="/car-crashes-weight-regression/01-combine-excel-files-across-multiple-sheets-and-save-as-csv-files/">        
2431      Combine Excel files across multiple sheets and save as CSV files
2432  </a>
2433  <div class="toc"></div>
2434
2435
2436
2437      </li>
2438      
2439      <li class="menu-item">
2440          
2441  <a href="/car-crashes-weight-regression/02-create-make-model-weights-csv/">        
2442      Create make model weights csv
2443  </a>
2444  <div class="toc"></div>
2445
2446
2447
2448      </li>
2449      
2450      <li class="menu-item">
2451          
2452  <a href="/car-crashes-weight-regression/03-find-car-data-from-vins/">        
2453      Find car data from VINs
2454  </a>
2455  <div class="toc"></div>
2456
2457
2458
2459      </li>
2460      
2461      <li class="menu-item">
2462          
2463  <a href="/car-crashes-weight-regression/04-combine-vins-and-weights/">        
2464      Combine VINs and weights
2465  </a>
2466  <div class="toc"></div>
2467
2468
2469
2470      </li>
2471      
2472      <li class="menu-item">
2473          
2474  <a href="/car-crashes-weight-regression/05-clean-combine-and-filter-data/">        
2475      Clean combine and filter data
2476  </a>
2477  <div class="toc"></div>
2478
2479
2480
2481      </li>
2482      
2483    </ol>
2484  </div>
2485
2486              </li>
2487            
2488              <li class="menu-item">
2489                
2490  <a href="/propublica-opportunity-gap/">        
2491      <strong>ProPublica:</strong>  Opportunity Gap
2492  </a>
2493  <div class="toc"></div>
2494
2495
2496
2497              </li>
2498            
2499              <li class="menu-item">
2500                
2501
2502
2503  <input id="accordion-strongbostonglobestrongticketingbias" type="checkbox" name="accordion-checkbox" hidden="">
2504  <label class="accordion-header c-hand" for="accordion-strongbostonglobestrongticketingbias">
2505      <i class="icon icon-arrow-right mr-1"></i>
2506      <strong>Boston Globe:</strong>  Ticketing bias
2507  </label>
2508  <div class="accordion-body">
2509      <ol class="menu menu-nav">
2510      
2511      <li class="menu-item">
2512          
2513  <a href="/boston-globe-tickets/">        
2514      Summary
2515  </a>
2516  <div class="toc"></div>
2517
2518
2519
2520      </li>
2521      
2522      <li class="menu-item">
2523          
2524  <a href="/boston-globe-tickets/boston-globe-ticketing-regression/">        
2525      Logistic regression for speeding tickets
2526  </a>
2527  <div class="toc"></div>
2528
2529
2530
2531      </li>
2532      
2533    </ol>
2534  </div>
2535
2536              </li>
2537            
2538              <li class="menu-item">
2539                
2540  <a href="/stanford-open-policing/">        
2541      <strong>Stanford:</strong>  Open Policing Data
2542  </a>
2543  <div class="toc"></div>
2544
2545
2546
2547              </li>
2548            
2549              <li class="menu-item">
2550                
2551
2552
2553  <input id="accordion-strongbuzzfeedstrongsurveillanceplanes" type="checkbox" name="accordion-checkbox" hidden="">
2554  <label class="accordion-header c-hand" for="accordion-strongbuzzfeedstrongsurveillanceplanes">
2555      <i class="icon icon-arrow-right mr-1"></i>
2556      <strong>BuzzFeed:</strong>  Surveillance planes
2557  </label>
2558  <div class="accordion-body">
2559      <ol class="menu menu-nav">
2560      
2561      <li class="menu-item">
2562          
2563  <a href="/buzzfeed-spy-planes/">        
2564      Summary
2565  </a>
2566  <div class="toc"></div>
2567
2568
2569
2570      </li>
2571      
2572      <li class="menu-item">
2573          
2574  <a href="/buzzfeed-spy-planes/feature-engineering-buzzfeed-spy-planes/">        
2575      Feature engineering - BuzzFeed spy planes
2576  </a>
2577  <div class="toc"></div>
2578
2579
2580
2581      </li>
2582      
2583      <li class="menu-item">
2584          
2585  <a href="/buzzfeed-spy-planes/drawing-flight-paths-on-maps-with-cartopy/">        
2586      Drawing flight paths on maps with cartopy
2587  </a>
2588  <div class="toc"></div>
2589
2590
2591
2592      </li>
2593      
2594      <li class="menu-item">
2595          
2596  <a href="/buzzfeed-spy-planes/buzzfeed-surveillance-planes-random-forests/">        
2597      Finding surveillance planes using random forests
2598  </a>
2599  <div class="toc"></div>
2600
2601
2602
2603      </li>
2604      
2605    </ol>
2606  </div>
2607
2608              </li>
2609            
2610              <li class="menu-item">
2611                
2612
2613
2614  <input id="accordion-strongrevealstrongmortgagelendingbias" type="checkbox" name="accordion-checkbox" hidden="">
2615  <label class="accordion-header c-hand" for="accordion-strongrevealstrongmortgagelendingbias">
2616      <i class="icon icon-arrow-right mr-1"></i>
2617      <strong>Reveal:</strong>  Mortgage lending bias
2618  </label>
2619  <div class="accordion-body">
2620      <ol class="menu menu-nav">
2621      
2622      <li class="menu-item">
2623          
2624  <a href="/reveal-mortgages/">        
2625      Summary
2626  </a>
2627  <div class="toc"></div>
2628
2629
2630
2631      </li>
2632      
2633      <li class="menu-item">
2634          
2635  <a href="/reveal-mortgages/reveal-mortgage-analysis-cleaning-and-combining-data/">        
2636      Cleaning and combining data for the Reveal Mortgage Analysis
2637  </a>
2638  <div class="toc"></div>
2639
2640
2641
2642      </li>
2643      
2644      <li class="menu-item">
2645          
2646  <a href="/reveal-mortgages/reveal-mortgage-analysis-wild-formulas-in-statsmodels-using-patsy-short-version/">        
2647      Wild formulas in statsmodels using Patsy (short version)
2648  </a>
2649  <div class="toc"></div>
2650
2651
2652
2653      </li>
2654      
2655      <li class="menu-item">
2656          
2657  <a href="/reveal-mortgages/reveal-mortgage-analysis-logistic-regression-using-statsmodels-formulas/">        
2658      Reveal Mortgage Analysis - Logistic Regression using statsmodels formulas
2659  </a>
2660  <div class="toc"></div>
2661
2662
2663
2664      </li>
2665      
2666      <li class="menu-item">
2667          
2668  <a href="/reveal-mortgages/reveal-mortgage-analysis-logistic-regression/">        
2669      Reveal Mortgage Analysis - Logistic Regression
2670  </a>
2671  <div class="toc"></div>
2672
2673
2674
2675      </li>
2676      
2677    </ol>
2678  </div>
2679
2680              </li>
2681            
2682              <li class="menu-item">
2683                
2684
2685
2686  <input id="accordion-strongapmreportsstrongjuryselectionbias" type="checkbox" name="accordion-checkbox" hidden="">
2687  <label class="accordion-header c-hand" for="accordion-strongapmreportsstrongjuryselectionbias">
2688      <i class="icon icon-arrow-right mr-1"></i>
2689      <strong>APM Reports:</strong>  Jury selection bias
2690  </label>
2691  <div class="accordion-body">
2692      <ol class="menu menu-nav">
2693      
2694      <li class="menu-item">
2695          
2696  <a href="/apm-reports-jury-bias/">        
2697      Summary
2698  </a>
2699  <div class="toc"></div>
2700
2701
2702
2703      </li>
2704      
2705      <li class="menu-item">
2706          
2707  <a href="/apm-reports-jury-bias/in-the-dark-combining-datasets-and-cleaning-the-data/">        
2708      Combining and cleaning the initial dataset
2709  </a>
2710  <div class="toc"></div>
2711
2712
2713
2714      </li>
2715      
2716      <li class="menu-item">
2717          
2718  <a href="/apm-reports-jury-bias/in-the-dark-feature-selection-with-p-values/">        
2719      Picking what matters and what doesn't in a regression
2720  </a>
2721  <div class="toc"></div>
2722
2723
2724
2725      </li>
2726      
2727      <li class="menu-item">
2728          
2729  <a href="/apm-reports-jury-bias/in-the-dark-jury-selection-regression-walkthrough/">        
2730      Analyzing data using statsmodels formulas
2731  </a>
2732  <div class="toc"></div>
2733
2734
2735
2736      </li>
2737      
2738      <li class="menu-item">
2739          
2740  <a href="/apm-reports-jury-bias/in-the-dark-alternative-formula-methods/">        
2741      Alternative techniques with statsmodels formulas
2742  </a>
2743  <div class="toc"></div>
2744
2745
2746
2747      </li>
2748      
2749    </ol>
2750  </div>
2751
2752              </li>
2753            
2754              <li class="menu-item">
2755                
2756
2757
2758  <input id="accordion-strongreutersstrongasylumdenials" type="checkbox" name="accordion-checkbox" hidden="">
2759  <label class="accordion-header c-hand" for="accordion-strongreutersstrongasylumdenials">
2760      <i class="icon icon-arrow-right mr-1"></i>
2761      <strong>Reuters:</strong>  Asylum denials
2762  </label>
2763  <div class="accordion-body">
2764      <ol class="menu menu-nav">
2765      
2766      <li class="menu-item">
2767          
2768  <a href="/reuters-asylum/">        
2769      Summary
2770  </a>
2771  <div class="toc"></div>
2772
2773
2774
2775      </li>
2776      
2777      <li class="menu-item">
2778          
2779  <a href="/reuters-asylum/cleaning-the-eoir-immigration-court-dataset/">        
2780      Preparing the EOIR immigration court data for analysis
2781  </a>
2782  <div class="toc"></div>
2783
2784
2785
2786      </li>
2787      
2788      <li class="menu-item">
2789          
2790  <a href="/reuters-asylum/using-regression-to-analyze-asylum-cases/">        
2791      How nationality and judges affect your chance of asylum in immigration court
2792  </a>
2793  <div class="toc"></div>
2794
2795
2796
2797      </li>
2798      
2799    </ol>
2800  </div>
2801
2802              </li>
2803            
2804              <li class="menu-item">
2805                
2806  <a href="/propublica-pardons/">        
2807      <strong>ProPublica:</strong>  Presidential pardons
2808  </a>
2809  <div class="toc"></div>
2810
2811
2812
2813              </li>
2814            
2815              <li class="menu-item">
2816                
2817
2818
2819  <input id="accordion-strongpropublicastrongcriminalsentencing" t
2819ype="checkbox" name="accordion-checkbox" hidden="">
2820  <label class="accordion-header c-hand" for="accordion-strongpropublicastrongcriminalsentencing">
2821      <i class="icon icon-arrow-right mr-1"></i>
2822      <strong>ProPublica:</strong>  Criminal sentencing
2823  </label>
2824  <div class="accordion-body">
2825      <ol class="menu menu-nav">
2826      
2827      <li class="menu-item">
2828          
2829  <a href="/propublica-criminal-sentencing/">        
2830      Summary
2831  </a>
2832  <div class="toc"></div>
2833
2834
2835
2836      </li>
2837      
2838      <li class="menu-item">
2839          
2840  <a href="/propublica-criminal-sentencing/week-5-1-machine-bias-class/">        
2841      Breaking down machine bias
2842  </a>
2843  <div class="toc"></div>
2844
2845
2846
2847      </li>
2848      
2849    </ol>
2850  </div>
2851
2852              </li>
2853            
2854              <li class="menu-item">
2855                
2856  <a href="/foia-predictor/">        
2857      <strong>data.world:</strong>  The FOIA Predictor
2858  </a>
2859  <div class="toc"></div>
2860
2861
2862
2863              </li>
2864            
2865          </ol>
2866        
2867    </div>
2868  </div>
2869</div>
2870      </div>
2871    </div>
2872  </div>
2873    
2873<script src="/js/tocbot.js"></script>
2873
2874
2875  
2875<script>
2876    // Add the div to hold the table of contents
2877    try {
2878      let toc = document.createElement("div")
2879      toc.setAttribute('class', 'js-toc')
2880      document.querySelector(".reading-options").parentNode.appendChild(toc)
2881    } catch (err) {
2882
2883    }
2884
2885    let slugSelector = "/wapo-app-reviews/scrape-app-store-reviews/"
2886    
2887      let contentSelector = '.notebook'
2888    
2889  </script>
2889
2890
2891  
2891<script src="/js/main.js?v=1edd6b72f7227d977df2bf125e3b6e55"></script>
2891
2892
2893</body>
2894
2895</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.