PageSourceSearch

https://investigate.ai/about/

html investigate.ai collected 2026-10-03 21:37:13 UTC 61,000 bytes, 2,225 lines download raw bytes

1
2  <!doctype html>
3<html lang="en-US">
4
5
6
7<head>
8  <!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 66 bytes, lines 8-9
8
9  <script async src="https://www.googletagmanager.com/gtag/js?id=
9UA-5541738-27
vendor: 14 bytes, lines 9-10
9"></script>
10  
10<script>
11    
vendor: 140 bytes, lines 11-15
11window.dataLayer = window.dataLayer || [];
12    function gtag(){dataLayer.push(arguments);}
13    gtag('js', new Date());
14
15    gtag('config', '
15UA-5541738-27
vendor: 6 bytes, lines 15-16
15');
16  
16</script>
16
17
18  <meta charset="utf-8">
19  <title>About investigate.ai | Data Science for Journalism</title>
20  <link href="/font-awesome/css/all.min.css?v=500d1a92f875b1d96d37a3a3f8f0438c" rel="stylesheet">
21  <!-- <link href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.7.5/css/bulma.min.css" rel="stylesheet">
22        <link href="https://fonts.googleapis.com/css?family=Raleway:400,700|Open+Sans:400,700&display=swap" rel="stylesheet"> -->
23  <link rel="stylesheet" href="/css/spectre.min.css?v=5cd401d486f79e82913923fe7d7f47ff">
24  <link rel="stylesheet" href="/css/spectre-exp.min.css?v=5909d80638a6ae6aa3a455b6f6a6d768">
25  <link rel="stylesheet" href="/css/spectre-icons.min.css?v=56b1bd38b79450b37939f8adb811d4cd">
26
27  <link rel="stylesheet" href="/css/tocbot.css?v=e8f0173e7c5216e5359587a88a570b77">
28
29  <!-- 
29<script src="https://unpkg.com/lunr/lunr.js"></script>
29 -->
30  
30<script src="/js/lunr.js?v=b93de9e9609074d666f5123e866e25ed"></script>
30
31  <!-- 
31<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>
31 -->
32  
32<script src="/js/require.min.js?v=18823f6a6d208ee1e361bb266ab794d5"></script>
32
33
34  <meta name="viewport" content="width=device-width, initial-scale=1">
35  <link href="/css/style.css?v=10420579fc48bad7c182d7d5e44a2cbb" rel="stylesheet">
36  <link href="/css/highlight.css?v=bdd372d828c6988b6071bb877211ebcb" rel="stylesheet">
37
38  <meta name="description" content="The incredibly storied history of an incredibly incredible machine learning website.">
39  <meta content="investigate.ai: Data Science for Journalists" property="og:site_name">
40
41  <meta name="twitter:creator" content="@dangerscarf">
42  <meta name="twitter:title" content="About investigate.ai | Data Science for Journalism">
43  <meta name="twitter:description" content="The incredibly storied history of an incredibly incredible machine learning website.">
44  <meta name="twitter:image" content="https://investigate.ai/images/flame/flame-searching.png">
45  <meta name="twitter:image:alt" content="A person metaphorically examining data">
46  <meta name="twitter:card" content="summary" />
47
48  <meta property="og:type" content="website" />
49  <meta property="og:url" content="https://investigate.ai/about/" />
50  <meta property="og:title" content="About investigate.ai | Data Science for Journalism" />
51  <meta property="og:description" content="The incredibly storied history of an incredibly incredible machine learning website." />
52  <meta property="og:image" content="https://investigate.ai/images/flame/flame-searching.png" />
53  <meta property="og:image:height" content="912" />
54  <meta property="og:image:width" content="921" />
55  <meta property="fb:admins" content="1504164"/>
56
57  <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
58  <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
59  <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
60  <link rel="manifest" href="/site.webmanifest">
61  <link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">
62  <meta name="msapplication-TileColor" content="#da532c">
63  <meta name="theme-color" content="#ffffff">
64
65</head>
66
67<body class="">
68  <div>
69      <div class="content-main">
70        <div class="nav-holder bg-secondary">
71  <div class='content'>
72    <header class="navbar">
73      <a href="#" class="sidebar-opener" aria-label="Sidebar opener"><i class="fas fa-bars fa-2x"></i></a>
74      <section class="navbar-section">
75        <a href="/" class="btn btn-link">Home</a>
76        <a href="/projects/" class="btn btn-link">Projects</a>
77        <a href="/topics/" class="btn btn-link">Topics</a>
78        <a href="/search/" class="btn btn-link">Search</a>
79        <a href="/newsletter/" class="btn btn-link">Newsletter</a>
80        <a href="/about/" class="btn btn-link">About</a>
81      </section>
82    </header>
83  </div>
84</div>
85        
86  
87    <div class="content">
88      <div class="columns callout">
89        <div class="column col-sm-auto col-5">
90            <img alt="logo for About investigate.ai" src="/images/flame/flame-family.png" style="padding: 2rem;">
91        </div>
92        <div class="column col-sm-auto col-7">
93            <h1>About investigate.ai</h1>
94            <p><p>I can't think of anything useful or clever to put here, so let me just say <a href="https://www.daringgourmet.com/traditional-nuernberger-elisenlebkuchen-german-lebkuchen/">this cookie recipe</a> is really good.</p></p>
95        </div>
96      </div>
97      <div id="page-content">
98        <p>Hi, I'm <a href="https://twitter.com/dangerscarf">Soma</a>!</p>
99<p>For the past four or five years I've run the <a href="http://ledeprogram.com/">Lede Program</a>, a non-degree summer program at Columbia's Journalism School that teaches journalists all sorts of data-y stuff. I'm also involved with the new <a href="https://journalism.columbia.edu/data">Data Journalism MS</a> program.</p>
100<p>They're both short programs, so we can't always cover all of the topics I'd like. <strong>Investigate.ai</strong> is me taking some of the fancier pieces, fleshing them out, and putting them on the internet to be cruelly ridiculed. Some of these pieces:</p>
101<ul>
102<li><a href="/reference/">Reference materials for machine learning</a></li>
103<li><a href="/topics/">Tutorials and walkthroughs</a></li>
104<li><a href="/projects/">Reproductions of journalism stories that have used machine learning</a> (or, yes, simple statistics)</li>
105</ul>
106<p>It was originally called "Data Science for Journalism," and while <strong>ds4j</strong> was incredibly cute the URL <strong>investigate.ai</strong> was just so cheesy enough to be infinitely memorable.</p>
107<h4>I have found a terrible, terrible error (or a much, much better way to do one of these things)</h4>
108<p><a href="https://meta.wikimedia.org/wiki/Cunningham%27s_Law">Thanks for letting me know</a>, you can find me at <a href="mailto:[email protected]">[email protected]</a> or <a href="http://twitter.com/dangerscarf">@dangerscarf</a>.</p>
109<h4>How should I learn Python?</h4>
110<p><a href="http://littlecolumns.com/learn/python/">I wrote a tutorial that I personally enjoy</a>, although it's more of the <em>basic</em> basics. After that you'll want to get into data analysis with the pandas library. Some people like <a href="https://www.firstpythonnotebook.org/">First Python Notebook</a>, so might as well give it a shot. If you'd like to burn a whole summer on this stuff <a href="http://ledeprogram.com/">you can always come hang out at Columbia</a>.</p>
111<h4>Machine learning isn't AI! And thus, investigate.ai is a terrible name.</h4>
112<p>¯\<em>(ツ)</em>/¯</p>
113<p>There's this whole issue with <code>.ml</code> domains so let's just accept that we now live in an <code>.ai</code> world. But if you'd prefer a hot take: machine learning and data science is statistics with a pay raise, and artificial intelligence is what the marketing departments calls anything involving a computer.</p>
114<p>(and at this moment I'm a one-man marketing department)</p>
115<h4>Where'd this all come from?</h4>
116<p>The Lede Program has had a class called <strong>Algorithms</strong> for the past few years, where we talk about how to perform machine learning as well as how algorithms make decisions all around us (e.g. filter bubbles, predictive policing, etc). It's been taught a lot of different ways by a handful of different instructors, and this is me building content around an approach that I think works well.</p>
117<h4>What's the approach?</h4>
118<p>Machine learning as a concept is seductive, and the best way to demystify it is to learn it and practice it. Instead of building up from math and first concepts, we put the tools to use building projects and try to talk a lot about what could go wrong. Journalists like doing stuff, so we give them stuff to do.</p>
119<p>Aspirationally, the students learn:</p>
120<ul>
121<li>Machine learning is not magic</li>
122<li>It's uncomfortably easy to do machine learning</li>
123<li>Machine learning often and easily easily produces garbage</li>
124<li>"But the computer did it" is not an excuse</li>
125</ul>
126<p>And that despite all this, machine learning can amazingly, actually, <em>sometimes</em> be useful.</p>
127<h4>Go on, please rant a little</h4>
128<p>I've tried to structure this material in a way that's hopefully a bit different than most machine learning courses. I could spend all the time in the world using words like eigenvalues and false negatives and f-1 scores but (I think, mostly) that... just isn't the important stuff to me, and it can be kind of alienating.</p>
129<p>I think the true decisions that you make when using machine learning are where you sit down and think about what your algorithm is doing and what happens when it's put into use. It helps you figure out the difference between an output of "an intern has to read more documents" vs "more people go to prison." Going from this practical angle provides, in my opinion, more responsible education than learning terms like "false positives."</p>
130<p>For example, <a href="https://course.elementsofai.com/">The Elements of AI Course</a> is delightful, but the "Implications" module gets thrown to the end. Same thing with the otherwise-wonderful <a href="https://course.fast.ai/">fast.ai</a> course - "Ethics" is just one part of the second-to-last lesson. </p>
131<p>
131It just seems like this stuff could be baked in earlier, and not necessarily separately. I get that "what are the implications of incorrectly identifying a Iris setosa as an Iris versicolor?" isn't the easiest thing in the world, but maybe the traditional datasets could use a refresh, too.</p>
132<p>(The <a href="http://euclid.psych.yorku.ca/SCS/Gallery/Private/anderson1936">1935 paper</a> the iris dataset is from has wicked visuals, though. And I don't think it's impossible to discuss what an ethics and responsible approach to classifying irises is, I just haven't done it yet!)</p>
133<h4>You seem somewhat dismissive of math</h4>
134<p>Math is great! <em>It does intimidate some people, though.</em></p>
135<p>Knowing how Bayes Theorem or linear algebra works is excellent for <em>some</em> aspects of optimization and determining the weaknesses of this or that approach or algorithm, but not knowing the details shouldn't stop you from building a tool to help search through <a href="https://www.washingtonpost.com/technology/2019/11/22/apple-says-its-app-store-is-safe-trusted-place-we-found-reports-unwanted-sexual-behavior-six-apps-some-targeting-minors/">130,000 app store reviews</a>.</p>
136<p><em>Understanding the possible side effects of your tools is more important than knowing how the tools work.</em> <a href="https://medium.com/@Soccermatics/how-practitioners-and-academics-think-and-then-forget-about-fairness-when-building-ai-systems-572530476982">Even experts are terrible at that</a>, even though the effects (including side effects) <em>are the reason we use machine learning in the first place</em>.</p>
137<p>And I mean, let's be honest, <a href="https://smile.amazon.com/Weapons-Math-Destruction-Increases-Inequality/dp/0553418815">I don't know if general adherence to the Math Gods has worked out so well</a>.</p>
138<h4>Why don't you talk about TensorFlow PyTorch BERT etc?</h4>
139<p>I'd like to, actually, especially since the Google Colab notebooks make it all super easy. I'm a big fan of people being able to run things on their own machines, though, so I figured we'd start with sklearn and build the rest on later.</p>
140<h4>I am very unhappy with your constant use of bold</h4>
141<p>I'm easily distracted and I find it an easier read! Helps me focus. <a href="https://twitter.com/Foone/status/1066547670477488128">Kinda like this</a>.</p>
142<h4 id="thankyou">Credits and thank-yous</h4>
143
144<p>Big thanks to <a href="https://journalism.columbia.edu/">Columbia's J-School</a> and <a href="https://www.knightfoundation.org/">The Knight Foundation</a>, who are the reasons this is a nice $free website.</p>
145<p>Infinite appreciation to the journalists who originally published the pieces reproduced here, especially those who took the time to chat with me, including 
146<a href="https://twitter.com/Aaron_Glantz">Aaron Glantz</a>, 
147<a href="https://twitter.com/bposton">Ben Poston</a>, 
148<a href="https://twitter.com/eman_thedataman">Emmanuel Martinez</a>, 
149<a href="https://twitter.com/JeffErnsthausen">Jeff Ernsthausen</a>, 
150<a href="https://twitter.com/j_la28">Jennifer LaFleur</a>, 
151<a href="https://twitter.com/jkeefe">John Keefe</a>, 
152<a href="https://twitter.com/Nat_Lash">Nathaniel Lash</a>, 
153<a href="https://twitter.com/nickyaforster">Nicky Forster</a>, 
154<a href="https://twitter.com/paldhous">Peter Aldhous</a>, 
155<a href="https://twitter.com/readelev">Ready Levinson</a>, 
156<a href="https://twitter.com/McNeill_Tweets">Ryan McNeill</a>, and 
157<a href="https://twitter.com/craftworksxyz">Will Craft</a>.</p>
158<p>A thousand thank-yours to the instructors for <a href="http://ledeprogram.com/">The Lede Program's</a> Algorithms course, for developing new content every year and making continual adjustments in the name of Making A Good Class:
159<a href="https://twitter.com/acookiecrumbles">Priyanjana Bengali</a>,
160<a href="https://twitter.com/chasedavis">Chase Davis</a>,
161<a href="https://www.linkedin.com/in/elizabet-doliar">Elizabeth Dollar</a>,
162<a href="https://twitter.com/rdunks1">Richard Dunks</a>,
163<a href="https://twitter.com/chrishwiggins">Chris Wiggins</a>, and
164<a href="https://twitter.com/jonathanstray">Jonathan Stray</a>.</p>
165<p>And all my students from <a href="http://ledeprogram.com/">Lede</a> and the <a href="https://journalism.columbia.edu/data">Data MS</a>, whose summers of suffering through learning Python made this all possible! Get ready for a very long list to show up here.</p>
166      </div>
167    </div>
168  
169
170        <div class="footer bg-secondary">
171  <div class="content">
172    <div class="columns">
173      <div class="column col-8 col-sm-12">
174        <p><strong>About the site</strong></p>
175        <p>Hi, I'm <a href="https://twitter.com/dangerscarf">Soma</a>, welcome to Data Science for Journalism a.k.a. investigate.ai!</p>
176        <p>There's been a lot of buzz about machine learning and "artificial intelligence" being used in stories over the past few years. It's mostly not that complicated - a little stats, a classifier here or there - but it's hard to know where to start without a little help.</p>
177        <p>If you know a little Python programming, hopefully this site can be that help! <a href="/about">Learn more about this project here.</a></p>
178        <p><strong>Our newsletter</strong></p>
179        <form action="https://littlecolumns.us12.list-manage.com/subscribe/post?u=ecdebf156be0b7e068fac7c25&amp;id=16537e7c90" method="post" role="form" target="_blank" novalidate>
180          <div class="input-group">
181            <input name="EMAIL" type="email" class="form-input" placeholder="Enter your Email" required="">
182            <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_ecdebf156be0b7e068fac7c25_16537e7c90" tabindex="-1" value=""></div>
183            <div class="input-group-append">
184              <button class="btn btn-primary input-group-btn" type="submit" aria-label="Email signup submit button">
185                <i class="fas fa-envelope"></i>
186                Sign up
187              </button>
188            </div>
189          </div>
190        </form>
191        
192      </div>
193      <div class="column col-4 col-sm-12">
194        <p><strong>Links</strong></p>
195        <ul>
196          <li><a href="mailto:[email protected]">[email protected]</a></li>
197          <li><a href="https://twitter.com/dangerscarf">@dangerscarf</a></li>
198          <li><a href="/privacy-policy/">Privacy policy</a></li>
199          <li><a href="/newsletter/">Newsletter</a></li>
200          <li>Images via <a href="https://icons8.com/">icons8</a></li>
201        </ul>
202        <p>Thanks to <a href="https://journalism.columbia.edu/">Columbia Journalism School</a>, the <a href="https://knightfoundation.org/">Knight Foundation</a>, and <a href="/about#thankyou">many others</a>.</p>
203      </div>
204    </div>
205  </div>
206</div>
207<script src="/js/scripts.js?v=e9d8cec8fc9221c9dd15f6e0314372b3" type="text/javascript"></script>
207
208      </div>
209      <div class="bg-dark content-sidebar">
210        <div class="sidebar-holder">
211  <div class="brand">
212    <a href="/">investigate.ai</a>
213    <br>
214    data science for everybody
215  </div>
216  
217  <div class="accordion-container">
218    <div class="accordion">
219      <div class="form-group sidebar-sticky">
220        <form method="GET" action="/search/" class="search-form">
221          <div class="input-group">
222            <input type="text" class="form-input input-sm" name="q" placeholder="Search investigate.ai">
223            <button class="btn btn-primary input-group-btn btn-sm">Search</button>
224          </div>
225          </form>
226      </div>
227    
228        
229          <h4 id="textanalysis" class="sidebar-sticky">
230            <a href="#textanalysis">Text analysis</a>
231          </h4>
232          <ol class="menu menu-nav">
233            
234              <li class="menu-item">
235                
236  <a href="/text-analysis/types-of-text-analysis/">        
237      Types of text analysis
238  </a>
239  <div class="toc"></div>
240
241
242
243              </li>
244            
245              <li class="menu-item">
246                
247
248
249  <input id="accordion-countingwords" type="checkbox" name="accordion-checkbox" hidden="">
250  <label class="accordion-header c-hand" for="accordion-countingwords">
251      <i class="icon icon-arrow-right mr-1"></i>
252      Counting words
253  </label>
254  <div class="accordion-body">
255      <ol class="menu menu-nav">
256      
257      <li class="menu-item">
258          
259  <a href="/text-analysis/counting-words-with-pythons-counter/">        
260      Simple word counting
261  </a>
262  <div class="toc"></div>
263
264
265
266      </li>
267      
268      <li class="menu-item">
269          
270  <a href="/text-analysis/counting-words-with-scikit-learns-countvectorizer/">        
271      Counting words across many documents
272  </a>
273  <div class="toc"></div>
274
275
276
277      </li>
278      
279      <li class="menu-item">
280          
281  <a href="/text-analysis/splitting-words-in-east-asian-languages/">        
282      Segmenting words in East Asian languages
283  </a>
284  <div class="toc"></div>
285
286
287
288      </li>
289      
290      <li class="menu-item">
291          
292  <a href="/caixin-museum-word-count/counting-words-in-chinese-museum-names/">        
293      Project: Caixin museums
294  </a>
295  <div class="toc"></div>
296
297
298
299      </li>
300      
301      <li class="menu-item">
302          
303  <a href="/text-analysis/how-to-make-scikit-learn-natural-language-processing-work-with-japanese-chinese/">        
304      Using scikit-learn vectorizers with East Asian languages
305  </a>
306  <div class="toc"></div>
307
308
309
310      </li>
311      
312    </ol>
313  </div>
314
315              </li>
316            
317              <li class="menu-item">
318                
319
320
321  <input id="accordion-advancedwordanalysis" type="checkbox" name="accordion-checkbox" hidden="">
322  <label class="accordion-header c-hand" for="accordion-advancedwordanalysis">
323      <i class="icon icon-arrow-right mr-1"></i>
324      Advanced word analysis
325  </label>
326  <div class="accordion-body">
327      <ol class="menu menu-nav">
328      
329      <li class="menu-item">
330          
331  <a href="/text-analysis/a-simple-explanation-of-tf-idf/">        
332      Upgraded word counts with TF-IDF
333  </a>
334  <div class="toc"></div>
335
336
337
338      </li>
339      
340      <li class="menu-item">
341          
342  <a href="/text-analysis/explaining-n-grams-in-natural-language-processing/">        
343      Multi-word phrases and n-grams
344  </a>
345  <div class="toc"></div>
346
347
348
349      </li>
350      
351      <li class="menu-item">
352          
353  <a href="/text-analysis/stemming-and-lemmatization/">        
354      Standardizing text with stemming and lemmatization
355  </a>
356  <div class="toc"></div>
357
358
359
360      </li>
361      
362      <li class="menu-item">
363          
364  <a href="/text-analysis/using-tf-idf-with-chinese/">        
365      Using TF-IDF with Chinese text
366  </a>
367  <div class="toc"></div>
368
369
370
371      </li>
372      
373    </ol>
374  </div>
375
376              </li>
377            
378              <li class="menu-item">
379                
380
381
382  <input id="accordion-sentimentanalysis" type="checkbox" name="accordion-checkbox" hidden="">
383  <label class="accordion-header c-hand" for="accordion-sentimentanalysis">
384      <i class="icon icon-arrow-right mr-1"></i>
385      Sentiment analysis
386  </label>
387  <div class="accordion-body">
388      <ol class="menu menu-nav">
389      
390      <li class="menu-item">
391          
392  <a href="/investigating-sentiment-analysis/comparing-sentiment-analysis-tools/">        
393      Comparing sentiment analysis tools
394  </a>
395  <div class="toc"></div>
396
397
398
399      </li>
400      
401      <li class="menu-item">
402          
403  <a href="/investigating-sentiment-analysis/designing-your-own-sentiment-analysis-tool/">        
404      Design your own sentiment analyzer
405  </a>
406  <div class="toc"></div>
407
408
409
410      </li>
411      
412      <li class="menu-item">
413          
414  <a href="/investigating-sentiment-analysis/more-data-to-train-our-sentiment-analysis-tool/">        
415      Improving your tool
416  </a>
417  <div class="toc"></div>
418
419
420
421      </li>
422      
423      <li class="menu-item">
424          
425  <a href="/upshot-trump-emolex/nrc-emotional-lexicon/">        
426      NRC Emotional Lexicon
427  </a>
428  <div class="toc"></div>
429
430
431
432      </li>
433      
434      <li class="menu-item">
435          
436  <a href="/upshot-trump-emolex/trump-vs-state-of-the-union-addresses/">        
437      Project: UpShot State of the Union
438  </a>
439  <div class="toc"></div>
440
441
442
443      </li>
444      
445      <li class="menu-item">
446          
447  <a href="/nyt-trump-tweets/">        
448      Project: NYT Trump tweets
449  </a>
450  <div class="toc"></div>
451
452
453
454      </li>
455      
456    </ol>
457  </div>
458
459              </li>
460            
461              <li class="menu-item">
462                
463
464
465  <input id="accordion-documentstotext" type="checkbox" name="accordion-checkbox" hidden="">
466  <label class="accordion-header c-hand" for="accordion-documentstotext">
467      <i class="icon icon-arrow-right mr-1"></i>
468      Documents to text
469  </label>
470  <div class="accordion-body">
471      <ol class="menu menu-nav">
472      
473      <li class="menu-item">
474          
475  <a href="/text-analysis/processing-documents-with-apache-tika/">        
476      Converting documents to text (English)
477  </a>
478  <div class="toc"></div>
479
480
481
482      </li>
483      
484      <li class="menu-item">
485          
486  <a href="/text-analysis/processing-documents-with-apache-tika-greek/">        
487      Converting documents to text (non-English)
488  </a>
489  <div class="toc"></div>
490
491
492
493      </li>
494      
495    </ol>
496  </div>
497
498              </li>
499            
500              <li class="menu-item">
501                
502
503
504  <input id="accordion-conceptspeopleandplaces" type="checkbox" name="accordion-checkbox" hidden="">
505  <label class="accordion-header c-hand" for="accordion-conceptspeopleandplaces">
506      <i class="icon icon-arrow-right mr-1"></i>
507      Concepts, people and places
508  </label>
509  <div class="accordion-body">
510      <ol class="menu menu-nav">
511      
512      <li class="menu-item">
513          
514  <a href="/text-analysis/introduction-to-topic-modeling/">        
515      Extracting topics from documents
516  </a>
517  <div class="toc"></div>
518
519
520
521      </li>
522      
523      <li class="menu-item">
524          
525  <a href="/text-analysis/choosing-the-right-number-of-topics-for-a-scikit-learn-topic-model/">        
526      Choosing the right number of topics
527  </a>
528  <div class="toc"></div>
529
530
531
532      </li>
533      
534      <li class="menu-item">
535          
536  <a href="/text-analysis/topic-models-with-gensim/">        
537      Topic models with Gensim
538  </a>
539  <div class="toc"></div>
540
541
542
543      </li>
544      
545      <li class="menu-item">
546          
547  <a href="/text-analysis/topic-modeling-and-clustering/">        
548      Topic models vs clustering
549  </a>
550  <div class="toc"></div>
551
552
553
554      </li>
555      
556      <li class="menu-item">
557          
558  <a href="/text-analysis/named-entity-recognition/">        
559      Entity recognition
560  </a>
561  <div class="toc"></div>
562
563
564
565      </li>
566      
567      <li class="menu-item">
568          
569  <a href="/text-analysis/word-embeddings/">        
570      Intro to word embeddings
571  </a>
572  <div class="toc"></div>
573
574
575
576      </li>
577      
578      <li class="menu-item">
579          
580  <a href="/text-analysis/document-similarity-using-word-embeddings/">        
581      Conceptual document similarity
582  </a>
583  <div class="toc"></div>
584
585
586
587      </li>
588      
589      <li class="menu-item">
590          
591  <a href="/text-analysis/comparing-documents-in-different-languages/">        
592      Comparing documents in different languages
593  </a>
594  <div class="toc"></div>
595
596
597
598      </li>
599      
600    </ol>
601  </div>
602
603              </li>
604            
605          </ol>
606        
607          <h4 id="puttingthingsincategoriesautomatically" class="sidebar-sticky">
608            <a href="#puttingthingsincategoriesautomatically">Putting things in categories automatically</a>
609          </h4>
610          <ol class="menu menu-nav">
611            
612              <li class="menu-item">
613                
614  <a href="/classification/intro-to-classification/">        
615      Introduction to Classification
616  </a>
617  <div class="toc"></div>
618
619
620
621              </li>
622            
623              <li class="menu-item">
624                
625
626
627  <input id="accordion-techniques" type="checkbox" name="accordion-checkbox" hidden="">
628  <label class="accordion-header c-hand" for="accordion-techniques">
629      <i class="icon icon-arrow-right mr-1"></i>
630      Techniques
631  </label>
632  <div class="accordion-body">
633      <ol class="menu menu-nav">
634      
635      <li class="menu-item">
636          
637  <a href="/classification/evaluating-classifiers/">        
638      Evaluating classifiers
639  </a>
640  <div class="toc"></div>
641
642
643
644      </li>
645      
646      <li class="menu-item">
647          
648  <a href="/classification/scikit-learn-and-categorical-features/">        
649      Categorical features
650  </a>
651  <div class="toc"></div>
652
653
654
655      </li>
656      
657      <li class="menu-item">
658          
659  <a href="/classification/using-classification-algorithms-with-text/">        
660      Classifiers with text
661  </a>
662  <div class="toc"></div>
663
664
665
666      </li>
667      
668      <li class="menu-item">
669          
670  <a href="/classification/correcting-for-imbalanced-datasets/">        
671      Correcting for imbalanced datasets
672  </a>
673  <div class="toc"></div>
674
675
676
677      </li>
678      
679    </ol>
680  </div>
681
682              </li>
683            
684              <li class="menu-item">
685                
686
687
688  <input id="accordion-projects" type="checkbox" name="accordion-checkbox" hidden="">
689  <label class="accordion-header c-hand" for="accordion-projects">
690      <i class="icon icon-arrow-right mr-1"></i>
691      Projects
692  </label>
693  <div class="accordion-body">
694      <ol class="menu menu-nav">
695      
696      <li class="menu-item">
697          
698  <a href="/buzzfeed-spy-planes/buzzfeed-surveillance-planes-random-forests/">        
699      BuzzFeed: Spy planes
700  </a>
701  <div class="toc"></div>
702
703
704
705      </li>
706      
707      <li class="menu-item">
708          
709  <a href="/wapo-app-reviews/predict-reviews/">        
710      WaPo chat: App reviews
711  </a>
712  <div class="toc"></div>
713
714
715
716      </li>
717      
718      <li class="menu-item">
719          
720  <a href="/nyt-takata-airbags/nyt-takata-completed/">        
721      NYT: Faulty airbag search
722  </a>
723  <div class="toc"></div>
724
725
726
727      </li>
728      
729      <li class="menu-item">
730          
731  <a href="/latimes-crime-classification/using-a-classifier-to-find-misclassified-crimes/">        
732      LA Times: crime classifier
733  </a>
734  <div class="toc"></div>
735
736
737
738      </li>
739      
740    </ol>
741  </div>
742
743              </li>
744            
745          </ol>
746        
747          <h4 id="howxaffectsy" class="sidebar-sticky">
748            <a href="#howxaffectsy">How X affects Y</a>
749          </h4>
750          <ol class="menu menu-nav">
751            
752              <li class="menu-item">
753                
754  <a href="/regression/what-is-regression/">        
755      Finding relationships with regression
756  </a>
757  <div class="toc"></div>
758
759
760
761              </li>
762            
763              <li class="menu-item">
764                
765
766
767  <input id="accordion-linearregression" type="checkbox" name="accordion-checkbox" hidden="">
768  <label class="accordion-header c-hand" for="accordion-linearregression">
769      <i class="icon icon-arrow-right mr-1"></i>
770      Linear Regression
771  </label>
772  <div class="accordion-body">
773      <ol class="menu menu-nav">
774      
775      <li class="menu-item">
776          
777  <a href="/regression/linear-regression-quickstart/">        
778      Linear regression (Quickstart)
779  </a>
780  <div class="toc"></div>
781
782
783
784      </li>
785      
786      <li class="menu-item">
787          
788  <a href="/regression/linear-regression/">        
789      Linear regression for humans
790  </a>
791  <div class="toc"></div>
792
793
794
795      </li>
796      
797      <li class="menu-item">
798          
799  <a href="/regression/linear-regression-part-two/">        
800      Putting regression to use
801  </a>
802  <div class="toc"></div>
803
804
805
806      </li>
807      
808      <li class="menu-item">
809          
810  <a href="/regression/linear-regression-evaluation/">        
811      Evaluating regressions
812  </a>
813  <div class="toc"></div>
814
815
816
817      </li>
818      
819      <li class="menu-item">
820          
821  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-formulas/">        
822      Associated Press: Life expectancy and unemployment
823  </a>
824  <div class="toc"></div>
825
826
827
828      </li>
829      
830    </ol>
831  </div>
832
833              </li>
834            
835              <li class="menu-item">
836                
837
838
839  <input id="accordion-logisticregression" type="checkbox" name="accordion-checkbox" hidden="">
840  <label class="accordion-header c-hand" for="accordion-logisticregression">
841      <i class="icon icon-arrow-right mr-1"></i>
842      Logistic Regression
843  </label>
844  <div class="accordion-body">
845      <ol class="menu menu-nav">
846      
847      <li class="menu-item">
848          
849  <a href="/regression/logistic-regression-quickstart/">        
850      Logistic regression (Quickstart)
851  </a>
852  <div class="toc"></div>
853
854
855
856      </li>
857      
858      <li class="menu-item">
859          
860  <a href="/regression/logistic-regression/">        
861      Logistic regression for humans
862  </a>
863  <div class="toc"></div>
864
865
866
867      </li>
868      
869      <li class="menu-item">
870          
871  <a href="/regression/logistic-regression-part-two/">        
872      More complex logistic regressions
873  </a>
874  <div class="toc"></div>
875
876
877
878      </li>
879      
880      <li class="menu-item">
881          
882  <a href="/regression/evaluating-logistic-regressions/">        
883      Evaluating logistic regressions
884  </a>
885  <div class="toc"></div>
886
887
888
889      </li>
890      
891      <li class="menu-item">
892          
893  <a href="/boston-globe-tickets/boston-globe-ticketing-regression/">        
894      Boston Globe: Speeding tickets
895  </a>
896  <div class="toc"></div>
897
898
899
900      </li>
901      
902      <li class="menu-item">
903          
904  <a href="/apm-reports-jury-bias/in-the-dark-alternative-formula-methods/">        
905      APM Reports: Jury selection
906  </a>
907  <div class="toc"></div>
908
909
910
911      </li>
912      
913    </ol>
914  </div>
915
916              </li>
917            
918          </ol>
919        
920          <h4 id="pythondatasciencereference" class="sidebar-sticky">
921            <a href="#pythondatasciencereference">Python data science reference</a>
922          </h4>
923          <ol class="menu menu-nav">
924            
925              <li class="menu-item">
926                
927  <a href="/reference/">        
928      Introduction
929  </a>
930  <div class="toc"></div>
931
932
933
934              </li>
935            
936              <li class="menu-item">
937                
938  <a href="/reference/vectorizing/">        
939      Vectorizing
940  </a>
941  <div class="toc"></div>
942
943
944
945              </li>
946            
947              <li class="menu-item">
948                
949  <a href="/reference/text-analysis/">        
950      Text Analysis
951  </a>
952  <div class="toc"></div>
953
954
955
956              </li>
957            
958              <li class="menu-item">
959                
960  <a href="/reference/regression/">        
961      Regression
962  </a>
963  <div class="toc"></div>
964
965
966
967              </li>
968            
969              <li class="menu-item">
970                
971  <a href="/reference/classification/">        
972      Classification
973  </a>
974  <div class="toc"></div>
975
976
977
978              </li>
979            
980          </ol>
981        
982          <h4 id="allprojects" class="sidebar-sticky">
983            <a href="#allprojects">All Projects</a>
984          </h4>
985          <ol class="menu menu-nav">
986            
987              <li class="menu-item">
988                
989  <a href="/projects/">        
990      Project Summaries
991  </a>
992  <div class="toc"></div>
993
994
995
996              </li>
997            
998              <li class="menu-item">
999                
1000
1001
1002  <input id="accordion-strongnytstrongtakataairbagsearch" type="checkbox" name="accordion-checkbox" hidden="">
1003  <label class="accordion-header c-hand" for="accordion-strongnytstrongtakataairbagsearch">
1004      <i class="icon icon-arrow-right mr-1"></i>
1005      <strong>NYT:</strong>  Takata airbag search
1006  </label>
1007  <div class="accordion-body">
1008      <ol class="menu menu-nav">
1009      
1010      <li class="menu-item">
1011          
1012  <a href="/nyt-takata-airbags/">        
1013      Summary
1014  </a>
1015  <div class="toc"></div>
1016
1017
1018
1019      </li>
1020      
1021      <li class="menu-item">
1022          
1023  <a href="/nyt-takata-airbags/airbag-classifier-search-binary/">        
1024      A simplistic reproduction of the NYT's research using logistic regression
1025  </a>
1026  <div class="toc"></div>
1027
1028
1029
1030      </li>
1031      
1032      <li class="menu-item">
1033          
1034  <a href="/nyt-takata-airbags/airbag-classifier-search-decision-tree/">        
1035      A decision-tree reproduction of the NYT's research
1036  </a>
1037  <div class="toc"></div>
1038
1039
1040
1041      </li>
1042      
1043      <li class="menu-item">
1044          
1045  <a href="/nyt-takata-airbags/airbag-classifier-search-countvectorizer/">        
1046      Combining a text vectorizer and a classifier to track down suspicious complaints
1047  </a>
1048  <div class="toc"></div>
1049
1050
1051
1052      </li>
1053      
1054    </ol>
1055  </div>
1056
1057              </li>
1058            
1059              <li class="menu-item">
1060                
1061
1062
1063  <input id="accordion-stronglatimesstrongcrimeclassification" type="checkbox" name="accordion-checkbox" hidden="">
1064  <label class="accordion-header c-hand" for="accordion-stronglatimesstrongcrimeclassification">
1065      <i class="icon icon-arrow-right mr-1"></i>
1066      <strong>LA Times:</strong>  Crime classification
1067  </label>
1068  <div class="accordion-body">
1069      <ol class="menu menu-nav">
1070      
1071      <li class="menu-item">
1072          
1073  <a href="/latimes-crime-classification/">        
1074      Summary
1075  </a>
1076  <div class="toc"></div>
1077
1078
1079
1080      </li>
1081      
1082      <li class="menu-item">
1083          
1084  <a href="/latimes-crime-classification/using-a-classifier-to-find-misclassified-crimes/">        
1085      Predicting downgraded assaults with machine learning
1086  </a>
1087  <div class="toc"></div>
1088
1089
1090
1091      </li>
1092      
1093      <li class="menu-item">
1094          
1095  <a href="/latimes-crime-classification/inspecting-classifications/">        
1096      Taking a closer look at our classifier and its misclassifications
1097  </a>
1098  <div class="toc"></div>
1099
1100
1101
1102      </li>
1103      
1104      <li class="menu-item">
1105          
1106  <a href="/latimes-crime-classification/trying-out-different-classifiers/">        
1107      Trying out and combining different classifiers
1108  </a>
1109  <div class="toc"></div>
1110
1111
1112
1113      </li>
1114      
1115    </ol>
1116  </div>
1117
1118              </li>
1119            
1120              <li class="menu-item">
1121                
1122
1123
1124  <input id="accordion-strongcaixinstrongmuseumnames" type="checkbox" name="accordion-checkbox" hidden="">
1125  <label class="accordion-header c-hand" for="accordion-strongcaixinstrongmuseumnames">
1126      <i class="icon icon-arrow-right mr-1"></i>
1127      <strong>Caixin:</strong>  Museum names
1128  </label>
1129  <div class="accordion-body">
1130      <ol class="menu menu-nav">
1131      
1132      <li class="menu-item">
1133          
1134  <a href="/caixin-museum-word-count/">        
1135      Summary
1136  </a>
1137  <div class="toc"></div>
1138
1139
1140
1141      </li>
1142      
1143      <li class="menu-item">
1144          
1145  <a href="/caixin-museum-word-count/chinese-museum-dataset-cleanup/">        
1146      Chinese museum dataset cleanup
1147  </a>
1148  <div class="toc"></div>
1149
1150
1151
1152      </li>
1153      
1154      <li class="menu-item">
1155          
1156  <a href="/caixin-museum-word-count/chinese-museums-per-capita-analysis/">        
1157      Chinese museums per capita analysis
1158  </a>
1159  <div class="toc"></div>
1160
1161
1162
1163      </li>
1164      
1165      <li class="menu-item">
1166          
1167  <a href="/caixin-museum-word-count/counting-words-in-chinese-museum-names/">        
1168      Counting words in Chinese museum names
1169  </a>
1170  <div class="toc"></div>
1171
1172
1173
1174      </li>
1175      
1176    </ol>
1177  </div>
1178
1179              </li>
1180            
1181              <li class="menu-item">
1182                
1183
1184
1185  <input id="accordion-strongwapostrongrandomchatappsafety" type="checkbox" name="accordion-checkbox" hidden="">
1186  <label class="accordion-header c-hand" for="accordion-strongwapostrongrandomchatappsafety">
1187      <i class="icon icon-arrow-right mr-1"></i>
1188      <strong>WaPo:</strong>  Random chat app safety
1189  </label>
1190  <div class="accordion-body">
1191      <ol class="menu menu-nav">
1192      
1193      <li class="menu-item">
1194          
1195  <a href="/wapo-app-reviews/">        
1196      Summary
1197  </a>
1198  <div class="toc"></div>
1199
1200
1201
1202      </li>
1203      
1204      <li class="menu-item">
1205          
1206  <a href="/wapo-app-reviews/scrape-app-store-reviews/">        
1207      Scrape and combine app store reviews
1208  </a>
1209  <div class="toc"></div>
1210
1211
1212
1213      </li>
1214      
1215      <li class="menu-item">
1216          
1217  <a href="/wapo-app-reviews/predict-reviews/">        
1218      Build a classifier to detect reviews about bad behavior
1219  </a>
1220  <div class="toc"></div>
1221
1222
1223
1224      </li>
1225      
1226    </ol>
1227  </div>
1228
1229              </li>
1230            
1231              <li class="menu-item">
1232                
1233  <a href="/ajc-doctors-abuse/">        
1234      <strong>AJC:</strong>  Doctors and sex abuse
1235  </a>
1236  <div class="toc"></div>
1237
1238
1239
1240              </li>
1241            
1242              <li class="menu-item">
1243                
1244
1245
1246  <input id="accordion-strongtheupshotstrongtrumpspeeches" type="checkbox" name="accordion-checkbox" hidden="">
1247  <label class="accordion-header c-hand" for="accordion-strongtheupshotstrongtrumpspeeches">
1248      <i class="icon icon-arrow-right mr-1"></i>
1249      <strong>The UpShot:</strong>  Trump speeches
1250  </label>
1251  <div class="accordion-body">
1252      <ol class="menu menu-nav">
1253      
1254      <li class="menu-item">
1255          
1256  <a href="/upshot-trump-emolex/">        
1257      Summary
1258  </a>
1259  <div class="toc"></div>
1260
1261
1262
1263      </li>
1264      
1265      <li class="menu-item">
1266          
1267  <a href="/upshot-trump-emolex/nrc-emotional-lexicon/">        
1268      An introduction to the NRC Emotional Lexicon
1269  </a>
1270  <div class="toc"></div>
1271
1272
1273
1274      </li>
1275      
1276      <li class="menu-item">
1277          
1278  <a href="/upshot-trump-emolex/trump-vs-state-of-the-union-addresses/">        
1279      Reproducing The UpShot's Trump State of the Union visualization
1280  </a>
1281  <div class="toc"></div>
1282
1283
1284
1285      </li>
1286      
1287    </ol>
1288  </div>
1289
1290              </li>
1291            
1292              <li class="menu-item">
1293                
1294
1295
1296  <input id="accordion-strongusatodaystrongmodellegislation" type="checkbox" name="accordion-checkbox" hidden="">
1297  <label class="accordion-header c-hand" for="accordion-strongusatodaystrongmodellegislation">
1298      <i class="icon icon-arrow-right mr-1"></i>
1299      <strong>USA Today:</strong>  Model legislation
1300  </label>
1301  <div class="accordion-body">
1302      <ol class="menu menu-nav">
1303      
1304      <li class="menu-item">
1305          
1306  <a href="/azcentral-text-reuse-model-legislation/">        
1307      Summary
1308  </a>
1309  <div class="toc"></div>
1310
1311
1312
1313      </li>
1314      
1315      <li class="menu-item">
1316          
1317  <a href="/azcentral-text-reuse-model-legislation/01-downloading-one-million-pieces-of-legislation-from-legiscan/">        
1318      Downloading one million pieces of legislation from LegiScan
1319  </a>
1320  <div class="toc"></div>
1321
1322
1323
1324      </li>
1325      
1326      <li class="menu-item">
1327          
1328  <a href="/azcentral-text-reuse-model-legislation/02-taking-a-mill
1328ion-pieces-of-legislation-from-a-csv-and-inserting-them-into-postgres/">        
1329      Taking a million pieces of legislation from a CSV and inserting them into Postgres
1330  </a>
1331  <div class="toc"></div>
1332
1333
1334
1335      </li>
1336      
1337      <li class="menu-item">
1338          
1339  <a href="/azcentral-text-reuse-model-legislation/03-download-word-pdf-and-html-content-and-process-it-into-text-with-tika/">        
1340      Download Word, PDF and HTML content and process it into text with Tika
1341  </a>
1342  <div class="toc"></div>
1343
1344
1345
1346      </li>
1347      
1348      <li class="menu-item">
1349          
1350  <a href="/azcentral-text-reuse-model-legislation/04-import-content-into-solr-for-advanced-text-searching/">        
1351      Import content into Solr for advanced text searching
1352  </a>
1353  <div class="toc"></div>
1354
1355
1356
1357      </li>
1358      
1359      <li class="menu-item">
1360          
1361  <a href="/azcentral-text-reuse-model-legislation/05-checking-for-legislative-text-reuse-using-python-solr-and-ngrams/">        
1362      Checking for legislative text reuse using Python, Solr, and ngrams
1363  </a>
1364  <div class="toc"></div>
1365
1366
1367
1368      </li>
1369      
1370      <li class="menu-item">
1371          
1372  <a href="/azcentral-text-reuse-model-legislation/05-checking-for-legislative-text-reuse-using-python-solr-and-simple-text-search/">        
1373      Checking for legislative text reuse using Python, Solr, and simple text search
1374  </a>
1375  <div class="toc"></div>
1376
1377
1378
1379      </li>
1380      
1381      <li class="menu-item">
1382          
1383  <a href="/azcentral-text-reuse-model-legislation/06-search-for-model-legislation-in-over-one-million-bills-using-postgres-and-solr/">        
1384      Search for model legislation in over one million bills using Postgres and Solr
1385  </a>
1386  <div class="toc"></div>
1387
1388
1389
1390      </li>
1391      
1392      <li class="menu-item">
1393          
1394  <a href="/azcentral-text-reuse-model-legislation/using-topic-modeling-to-categorize-legislation/">        
1395      Using topic modeling to categorize legislation
1396  </a>
1397  <div class="toc"></div>
1398
1399
1400
1401      </li>
1402      
1403    </ol>
1404  </div>
1405
1406              </li>
1407            
1408              <li class="menu-item">
1409                
1410  <a href="/fcc-comments/">        
1411      FCC comment bots
1412  </a>
1413  <div class="toc"></div>
1414
1415
1416
1417              </li>
1418            
1419              <li class="menu-item">
1420                
1421
1422
1423  <input id="accordion-strongbloombergstrongdemocraticcandidatetweets" type="checkbox" name="accordion-checkbox" hidden="">
1424  <label class="accordion-header c-hand" for="accordion-strongbloombergstrongdemocraticcandidatetweets">
1425      <i class="icon icon-arrow-right mr-1"></i>
1426      <strong>Bloomberg:</strong>  Democratic Candidate Tweets
1427  </label>
1428  <div class="accordion-body">
1429      <ol class="menu menu-nav">
1430      
1431      <li class="menu-item">
1432          
1433  <a href="/bloomberg-tweet-topics/">        
1434      Summary
1435  </a>
1436  <div class="toc"></div>
1437
1438
1439
1440      </li>
1441      
1442      <li class="menu-item">
1443          
1444  <a href="/bloomberg-tweet-topics/scrape-tweets-from-presidential-primary-candidates/">        
1445      Downloading all 2019 tweets from Democratic presidential candidates
1446  </a>
1447  <div class="toc"></div>
1448
1449
1450
1451      </li>
1452      
1453      <li class="menu-item">
1454          
1455  <a href="/bloomberg-tweet-topics/topic-modeling-for-tweets/">        
1456      Using topic modeling to analyze presidential candidate tweets
1457  </a>
1458  <div class="toc"></div>
1459
1460
1461
1462      </li>
1463      
1464      <li class="menu-item">
1465          
1466  <a href="/bloomberg-tweet-topics/assigning-categories-to-text-using-keyword-matching/">        
1467      Assigning categories to tweets using keyword matching
1468  </a>
1469  <div class="toc"></div>
1470
1471
1472
1473      </li>
1474      
1475      <li class="menu-item">
1476          
1477  <a href="/bloomberg-tweet-topics/building-streamgraphs-from-candidate-tweets/">        
1478      Building streamgraphs from categorized and dated datasets
1479  </a>
1480  <div class="toc"></div>
1481
1482
1483
1484      </li>
1485      
1486    </ol>
1487  </div>
1488
1489              </li>
1490            
1491              <li class="menu-item">
1492                
1493  <a href="/nyt-trump-tweets/">        
1494      <strong>NYT:</strong>  Trump tweets
1495  </a>
1496  <div class="toc"></div>
1497
1498
1499
1500              </li>
1501            
1502              <li class="menu-item">
1503                
1504
1505
1506  <input id="accordion-strongapstronglifeexpectancy" type="checkbox" name="accordion-checkbox" hidden="">
1507  <label class="accordion-header c-hand" for="accordion-strongapstronglifeexpectancy">
1508      <i class="icon icon-arrow-right mr-1"></i>
1509      <strong>AP:</strong>  Life expectancy
1510  </label>
1511  <div class="accordion-body">
1512      <ol class="menu menu-nav">
1513      
1514      <li class="menu-item">
1515          
1516  <a href="/ap-regression-unemployment/">        
1517      Summary
1518  </a>
1519  <div class="toc"></div>
1520
1521
1522
1523      </li>
1524      
1525      <li class="menu-item">
1526          
1527  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-formulas/">        
1528      Simple logistic regression using statsmodels (formula version)
1529  </a>
1530  <div class="toc"></div>
1531
1532
1533
1534      </li>
1535      
1536      <li class="menu-item">
1537          
1538  <a href="/ap-regression-unemployment/simple-regression-with-census-data-statsmodels-with-dataframes/">        
1539      Simple logistic regression using statsmodels (dataframes version)
1540  </a>
1541  <div class="toc"></div>
1542
1543
1544
1545      </li>
1546      
1547    </ol>
1548  </div>
1549
1550              </li>
1551            
1552              <li class="menu-item">
1553                
1554  <a href="/fivethirtyeight-p-hacking/">        
1555      <strong>FiveThirtyEight:</strong>  P-values
1556  </a>
1557  <div class="toc"></div>
1558
1559
1560
1561              </li>
1562            
1563              <li class="menu-item">
1564                
1565
1566
1567  <input id="accordion-strongmilwaukeejournalsentinelstrongpotholes" type="checkbox" name="accordion-checkbox" hidden="">
1568  <label class="accordion-header c-hand" for="accordion-strongmilwaukeejournalsentinelstrongpotholes">
1569      <i class="icon icon-arrow-right mr-1"></i>
1570      <strong>Milwaukee Journal-Sentinel:</strong>  Potholes
1571  </label>
1572  <div class="accordion-body">
1573      <ol class="menu menu-nav">
1574      
1575      <li class="menu-item">
1576          
1577  <a href="/milwaukee-potholes/">        
1578      Summary
1579  </a>
1580  <div class="toc"></div>
1581
1582
1583
1584      </li>
1585      
1586      <li class="menu-item">
1587          
1588  <a href="/milwaukee-potholes/milwaukee-journal-sentinel-and-potholes-full-walkthrough/">        
1589      Pothole geographic analysis and linear regression, complete walkthrough
1590  </a>
1591  <div class="toc"></div>
1592
1593
1594
1595      </li>
1596      
1597      <li class="menu-item">
1598          
1599  <a href="/milwaukee-potholes/milwaukee-journal-sentinel-and-potholes-without-merging/">        
1600      Pothole demographics linear regression, no spatial analysis
1601  </a>
1602  <div class="toc"></div>
1603
1604
1605
1606      </li>
1607      
1608    </ol>
1609  </div>
1610
1611              </li>
1612            
1613              <li class="menu-item">
1614                
1615
1616
1617  <input id="accordion-strongdallasmorningnewsstrongcheatingschools" type="checkbox" name="accordion-checkbox" hidden="">
1618  <label class="accordion-header c-hand" for="accordion-strongdallasmorningnewsstrongcheatingschools">
1619      <i class="icon icon-arrow-right mr-1"></i>
1620      <strong>Dallas Morning News:</strong>  Cheating schools
1621  </label>
1622  <div class="accordion-body">
1623      <ol class="menu menu-nav">
1624      
1625      <li class="menu-item">
1626          
1627  <a href="/dmn-texas-school-cheating/">        
1628      Summary
1629  </a>
1630  <div class="toc"></div>
1631
1632
1633
1634      </li>
1635      
1636      <li class="menu-item">
1637          
1638  <a href="/dmn-texas-school-cheating/texas-school-cheating-finding-outliers-with-standard-deviation-and-regression/">        
1639      Finding outliers with standard deviation and regression
1640  </a>
1641  <div class="toc"></div>
1642
1643
1644
1645      </li>
1646      
1647      <li class="menu-item">
1648          
1649  <a href="/dmn-texas-school-cheating/texas-school-cheating-finding-outliers-with-regression-residuals/">        
1650      Finding outliers with regression residuals (short version)
1651  </a>
1652  <div class="toc"></div>
1653
1654
1655
1656      </li>
1657      
1658      <li class="menu-item">
1659          
1660  <a href="/dmn-texas-school-cheating/texas-school-cheating-graph-reproductions/">        
1661      Reproducing the graphics from The Dallas Morning News piece
1662  </a>
1663  <div class="toc"></div>
1664
1665
1666
1667      </li>
1668      
1669    </ol>
1670  </div>
1671
1672              </li>
1673            
1674              <li class="menu-item">
1675                
1676
1677
1678  <input id="accordion-strongtampabaytimesstrongfailurefactories" type="checkbox" name="accordion-checkbox" hidden="">
1679  <label class="accordion-header c-hand" for="accordion-strongtampabaytimesstrongfailurefactories">
1680      <i class="icon icon-arrow-right mr-1"></i>
1681      <strong>Tampa Bay Times:</strong>  Failure factories
1682  </label>
1683  <div class="accordion-body">
1684      <ol class="menu menu-nav">
1685      
1686      <li class="menu-item">
1687          
1688  <a href="/tampa-bay-times-schools/">        
1689      Summary
1690  </a>
1691  <div class="toc"></div>
1692
1693
1694
1695      </li>
1696      
1697      <li class="menu-item">
1698          
1699  <a href="/tampa-bay-times-schools/linear-regression-on-florida-schools/">        
1700      Linear regression on Florida schools, complete walkthrough
1701  </a>
1702  <div class="toc"></div>
1703
1704
1705
1706      </li>
1707      
1708      <li class="menu-item">
1709          
1710  <a href="/tampa-bay-times-schools/linear-regression-on-florida-schools-no-cleaning/">        
1711      Linear regression on Florida schools, no cleaning
1712  </a>
1713  <div class="toc"></div>
1714
1715
1716
1717      </li>
1718      
1719    </ol>
1720  </div>
1721
1722              </li>
1723            
1724              <li class="menu-item">
1725                
1726
1727
1728  <input id="accordion-caraccidentsandcarweight" type="checkbox" name="accordion-checkbox" hidden="">
1729  <label class="accordion-header c-hand" for="accordion-caraccidentsandcarweight">
1730      <i class="icon icon-arrow-right mr-1"></i>
1731      Car accidents and car weight
1732  </label>
1733  <div class="accordion-body">
1734      <ol class="menu menu-nav">
1735      
1736      <li class="menu-item">
1737          
1738  <a href="/car-crashes-weight-regression/">        
1739      Summary
1740  </a>
1741  <div class="toc"></div>
1742
1743
1744
1745      </li>
1746      
1747      <li class="menu-item">
1748          
1749  <a href="/car-crashes-weight-regression/car-crashes-feature-selection-and-engineering/">        
1750      Feature selection and engineering
1751  </a>
1752  <div class="toc"></div>
1753
1754
1755
1756      </li>
1757      
1758      <li class="menu-item">
1759          
1760  <a href="/car-crashes-weight-regression/01-combine-excel-files-across-multiple-sheets-and-save-as-csv-files/">        
1761      Combine Excel files across multiple sheets and save as CSV files
1762  </a>
1763  <div class="toc"></div>
1764
1765
1766
1767      </li>
1768      
1769      <li class="menu-item">
1770          
1771  <a href="/car-crashes-weight-regression/02-create-make-model-weights-csv/">        
1772      Create make model weights csv
1773  </a>
1774  <div class="toc"></div>
1775
1776
1777
1778      </li>
1779      
1780      <li class="menu-item">
1781          
1782  <a href="/car-crashes-weight-regression/03-find-car-data-from-vins/">        
1783      Find car data from VINs
1784  </a>
1785  <div class="toc"></div>
1786
1787
1788
1789      </li>
1790      
1791      <li class="menu-item">
1792          
1793  <a href="/car-crashes-weight-regression/04-combine-vins-and-weights/">        
1794      Combine VINs and weights
1795  </a>
1796  <div class="toc"></div>
1797
1798
1799
1800      </li>
1801      
1802      <li class="menu-item">
1803          
1804  <a href="/car-crashes-weight-regression/05-clean-combine-and-filter-data/">        
1805      Clean combine and filter data
1806  </a>
1807  <div class="toc"></div>
1808
1809
1810
1811      </li>
1812      
1813    </ol>
1814  </div>
1815
1816              </li>
1817            
1818              <li class="menu-item">
1819                
1820  <a href="/propublica-opportunity-gap/">        
1821      <strong>ProPublica:</strong>  Opportunity Gap
1822  </a>
1823  <div class="toc"></div>
1824
1825
1826
1827              </li>
1828            
1829              <li class="menu-item">
1830                
1831
1832
1833  <input id="accordion-strongbostonglobestrongticketingbias" type="checkbox" name="accordion-checkbox" hidden="">
1834  <label class="accordion-header c-hand" for="accordion-strongbostonglobestrongticketingbias">
1835      <i class="icon icon-arrow-right mr-1"></i>
1836      <strong>Boston Globe:</strong>  Ticketing bias
1837  </label>
1838  <div class="accordion-body">
1839      <ol class="menu menu-nav">
1840      
1841      <li class="menu-item">
1842          
1843  <a href="/boston-globe-tickets/">        
1844      Summary
1845  </a>
1846  <div class="toc"></div>
1847
1848
1849
1850      </li>
1851      
1852      <li class="menu-item">
1853          
1854  <a href="/boston-globe-tickets/boston-globe-ticketing-regression/">        
1855      Logistic regression for speeding tickets
1856  </a>
1857  <div class="toc"></div>
1858
1859
1860
1861      </li>
1862      
1863    </ol>
1864  </div>
1865
1866              </li>
1867            
1868              <li class="menu-item">
1869                
1870  <a href="/stanford-open-policing/">        
1871      <strong>Stanford:</strong>  Open Policing Data
1872  </a>
1873  <div class="toc"></div>
1874
1875
1876
1877              </li>
1878            
1879              <li class="menu-item">
1880                
1881
1882
1883  <input id="accordion-strongbuzzfeedstrongsurveillanceplanes" type="checkbox" name="accordion-checkbox" hidden="">
1884  <label class="accordion-header c-hand" for="accordion-strongbuzzfeedstrongsurveillanceplanes">
1885      <i class="icon icon-arrow-right mr-1"></i>
1886      <strong>BuzzFeed:</strong>  Surveillance planes
1887  </label>
1888  <div class="accordion-body">
1889      <ol class="menu menu-nav">
1890      
1891      <li class="menu-item">
1892          
1893  <a href="/buzzfeed-spy-planes/">        
1894      Summary
1895  </a>
1896  <div class="toc"></div>
1897
1898
1899
1900      </li>
1901      
1902      <li class="menu-item">
1903          
1904  <a href="/buzzfeed-spy-planes/feature-engineering-buzzfeed-spy-planes/">        
1905      Feature engineering - BuzzFeed spy planes
1906  </a>
1907  <div class="toc"></div>
1908
1909
1910
1911      </li>
1912      
1913      <li class="menu-item">
1914          
1915  <a href="/buzzfeed-spy-planes/drawing-flight-paths-on-maps-with-cartopy/">        
1916      Drawing flight paths on maps with cartopy
1917  </a>
1918  <div class="toc"></div>
1919
1920
1921
1922      </li>
1923      
1924      <li class="menu-item">
1925          
1926  <a href="/buzzfeed-spy-planes/buzzfeed-surveillance-planes-random-forests/">        
1927      Finding surveillance planes using random forests
1928  </a>
1929  <div class="toc"></div>
1930
1931
1932
1933      </li>
1934      
1935    </ol>
1936  </div>
1937
1938              </li>
1939            
1940              <li class="menu-item">
1941                
1942
1943
1944  <input id="accordion-strongrevealstrongmortgagelendingbias" type="checkbox" name="accordion-checkbox" hidden="">
1945  <label class="accordion-header c-hand" for="accordion-strongrevealstrongmortgagelendingbias">
1946      <i class="icon icon-arrow-right mr-1"></i>
1947      <strong>Reveal:</strong>  Mortgage lending bias
1948  </label>
1949  <div class="accordion-body">
1950      <ol class="menu menu-nav">
1951      
1952      <li class="menu-item">
1953          
1954  <a href="/reveal-mortgages/">        
1955      Summary
1956  </a>
1957  <div class="toc"></div>
1958
1959
1960
1961      </li>
1962      
1963      <li class="menu-item">
1964          
1965  <a href="/reveal-mortgages/reveal-mortgage-analysis-cleaning-and-combining-data/">        
1966      Cleaning and combining data for the Reveal Mortgage Analysis
1967  </a>
1968  <div class="toc"></div>
1969
1970
1971
1972      </li>
1973      
1974      <li class="menu-item">
1975          
1976  <a href="/reveal-mortgages/reveal-mortgage-analysis-wild-formulas-in-statsmodels-using-patsy-short-version/">        
1977      Wild formulas in statsmodels using Patsy (short version)
1978  </a>
1979  <div class="toc"></div>
1980
1981
1982
1983      </li>
1984      
1985      <li class="menu-item">
1986          
1987  <a href="/reveal-mortgages/reveal-mortgage-analysis-logistic-regression-using-statsmodels-formulas/">        
1988      Reveal Mortgage Analysis - Logistic Regression using statsmodels formulas
1989  </a>
1990  <div class="toc"></div>
1991
1992
1993
1994      </li>
1995      
1996      <li class="menu-item">
1997          
1998  <a href="/reveal-mortgages/reveal-mortgage-analysis-logistic-regression/">        
1999      Reveal Mortgage Analysis - Logistic Regression
2000  </a>
2001  <div class="toc"></div>
2002
2003
2004
2005      </li>
2006      
2007    </ol>
2008  </div>
2009
2010              </li>
2011            
2012              <li class="menu-item">
2013                
2014
2015
2016  <input id="accordion-strongapmreportsstrongjuryselectionbias" type="checkbox" name="accordion-checkbox" hidden="">
2017  <label class="accordion-header c-hand" for="accordion-strongapmreportsstrongjuryselectionbias">
2018      <i class="icon icon-arrow-right mr-1"></i>
2019      <strong>APM Reports:</strong>  Jury selection bias
2020  </label>
2021  <div class="accordion-body">
2022      <ol class="menu menu-nav">
2023      
2024      <li class="menu-item">
2025          
2026  <a href="/apm-reports-jury-bias/">        
2027      Summary
2028  </a>
2029  <div class="toc"></div>
2030
2031
2032
2033      </li>
2034      
2035      <li class="menu-item">
2036          
2037  <a href="/apm-reports-jury-bias/in-the-dark-combining-datasets-and-cleaning-the-data/">        
2038      Combining and cleaning the initial dataset
2039  </a>
2040  <div class="toc"></div>
2041
2042
2043
2044      </li>
2045      
2046      <li class="menu-item">
2047          
2048  <a href="/apm-reports-jury-bias/in-the-dark-feature-selection-with-p-values/">        
2049      Picking what matters and what doesn't in a regression
2050  </a>
2051  <div class="toc"></div>
2052
2053
2054
2055      </li>
2056      
2057      <li class="menu-item">
2058          
2059  <a href="/apm-reports-jury-bias/in-the-dark-jury-selection-regression-walkthrough/">        
2060      Analyzing data using statsmodels formulas
2061  </a>
2062  <div class="toc"></div>
2063
2064
2065
2066      </li>
2067      
2068      <li class="menu-item">
2069          
2070  <a href="/apm-reports-jury-bias/in-the-dark-alternative-formula-methods/">        
2071      Alternative techniques with statsmodels formulas
2072  </a>
2073  <div class="toc"></div>
2074
2075
2076
2077      </li>
2078      
2079    </ol>
2080  </div>
2081
2082              </li>
2083            
2084              <li class="menu-item">
2085                
2086
2087
2088  <input id="accordion-strongreutersstrongasylumdenials" type="checkbox" name="accordion-checkbox" hidden="">
2089  <label class="accordion-header c-hand" for="accordion-strongreutersstrongasylumdenials">
2090      <i class="icon icon-arrow-right mr-1"></i>
2091      <strong>Reuters:</strong>  Asylum denials
2092  </label>
2093  <div class="accordion-body">
2094      <ol class="menu menu-nav">
2095      
2096      <li class="menu-item">
2097          
2098  <a href="/reuters-asylum/">        
2099      Summary
2100  </a>
2101  <div class="toc"></div>
2102
2103
2104
2105      </li>
2106      
2107      <li class="menu-item">
2108          
2109  <a href="/reuters-asylum/cleaning-the-eoir-immigration-court-dataset/">        
2110      Preparing the EOIR immigration court data for analysis
2111  </a>
2112  <div class="toc"></div>
2113
2114
2115
2116      </li>
2117      
2118      <li class="menu-item">
2119          
2120  <a href="/reuters-asylum/using-regression-to-analyze-asylum-cases/">        
2121      How nationality and judges affect your chance of asylum in immigration court
2122  </a>
2123  <div class="toc"></div>
2124
2125
2126
2127      </li>
2128      
2129    </ol>
2130  </div>
2131
2132              </li>
2133            
2134              <li class="menu-item">
2135                
2136  <a href="/propublica-pardons/">        
2137      <strong>ProPublica:</strong>  Presidential pardons
2138  </a>
2139  <div class="toc"></div>
2140
2141
2142
2143              </li>
2144            
2145              <li class="menu-item">
2146                
2147
2148
2149  <input id="accordion-strongpropublicastrongcriminalsentencing" t
2149ype="checkbox" name="accordion-checkbox" hidden="">
2150  <label class="accordion-header c-hand" for="accordion-strongpropublicastrongcriminalsentencing">
2151      <i class="icon icon-arrow-right mr-1"></i>
2152      <strong>ProPublica:</strong>  Criminal sentencing
2153  </label>
2154  <div class="accordion-body">
2155      <ol class="menu menu-nav">
2156      
2157      <li class="menu-item">
2158          
2159  <a href="/propublica-criminal-sentencing/">        
2160      Summary
2161  </a>
2162  <div class="toc"></div>
2163
2164
2165
2166      </li>
2167      
2168      <li class="menu-item">
2169          
2170  <a href="/propublica-criminal-sentencing/week-5-1-machine-bias-class/">        
2171      Breaking down machine bias
2172  </a>
2173  <div class="toc"></div>
2174
2175
2176
2177      </li>
2178      
2179    </ol>
2180  </div>
2181
2182              </li>
2183            
2184              <li class="menu-item">
2185                
2186  <a href="/foia-predictor/">        
2187      <strong>data.world:</strong>  The FOIA Predictor
2188  </a>
2189  <div class="toc"></div>
2190
2191
2192
2193              </li>
2194            
2195          </ol>
2196        
2197    </div>
2198  </div>
2199</div>
2200      </div>
2201    </div>
2202  </div>
2203    
2203<script src="/js/tocbot.js"></script>
2203
2204
2205  
2205<script>
2206    // Add the div to hold the table of contents
2207    try {
2208      let toc = document.createElement("div")
2209      toc.setAttribute('class', 'js-toc')
2210      document.querySelector(".reading-options").parentNode.appendChild(toc)
2211    } catch (err) {
2212
2213    }
2214
2215    let slugSelector = "/about/"
2216    
2217      let contentSelector = '.content'
2218    
2219  </script>
2219
2220
2221  
2221<script src="/js/main.js?v=1edd6b72f7227d977df2bf125e3b6e55"></script>
2221
2222
2223</body>
2224
2225</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.