PageSourceSearch

https://nathanmarz.com/blog/?currentPage=2

html nathanmarz.com collected 2026-09-24 17:59:23 UTC 107,462 bytes, 1,631 lines download raw bytes

1<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
2
3<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
4
5<head>
6  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> 
7  
8   <meta name="Robots" content="index,follow" />  
9  
10  
11  <!-- nmarz -->
12
13  <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
14
15  <link id="primaryStylesheet" rel="stylesheet" type="text/css" href="/display/common.css?styleId=803050&amp;RK=1464455086687&amp;CE=75"/>
16  
17  <link id="slideShowStylesheet" rel="stylesheet" type="text/css" href="/universal/styles/squarespace-gallery-slideshow.css?CE=75"/>
18  
19  
20    <link rel="alternate" type="application/rss+xml" title="RSS" href="http://feeds.feedburner.com/ThoughtsFromTheRedPlanet" />
21  
22    <link rel="EditURI" type="application/rsd+xml" title="RSD" href="http://nathanmarz.com/blog/rsd.xml" />
23  
24  
25  <!-- Google Web Fonts -->
26  
26<script type="text/javascript">
27    	WebFontConfig = {};
28  
29   
30    (function() {
31      var wf = document.createElement('script');
32      wf.src = ('https:' == document.location.protocol ? 'https' : 'http') +
33          '://ajax.googleapis.com/ajax/libs/webfont/1/webfont.js';
34      wf.type = 'text/javascript';
35      wf.async = 'true';
36      var s = document.getElementsByTagName('script')[0];
37      s.parentNode.insertBefore(wf, s);
38    })();
39  </script>
39
40
41  
41<script type="text/javascript" src="/universal/yui/yahoo-dom-event/yahoo-dom-event.js?CE=75"></script>
vendor: 1 bytes, line 41
41
42<script type="text/javascript" src="/universal/yui/connection/connection_core-min.js?CE=75"></script>
vendor: 1 bytes, line 42
42
43<script type="text/javascript" src="/universal/yui/json/json-min.js?CE=75"></script>
vendor: 1 bytes, line 43
43
44<script type="text/javascript" src="/universal/yui/animation/animation-min.js?CE=75"></script>
vendor: 1 bytes, line 44
44
45<script type="text/javascript" src="/universal/yui/selector/selector.js?CE=75"></script>
vendor: 1 bytes, line 45
45
46<script type="text/javascript" src="/universal/yui/element/element.js?CE=75"></script>
vendor: 1 bytes, line 46
46
47<script type="text/javascript" src="/universal/scripts/global.js?CE=75"></script>
vendor: 1 bytes, line 47
47
48<script type="text/javascript" src="/universal/scripts/lightbox.js?CE=75"></script>
vendor: 1 bytes, line 48
48
49<script type="text/javascript" src="/universal/scripts/api.js?CE=75"></script>
vendor: 1 bytes, line 49
49
50<script type="text/javascript" src="/universal/scripts/base64.js?CE=75"></script>
vendor: 1 bytes, line 50
50
51<script type="text/javascript" src="/universal/scripts/census.js?CE=75"></script>
vendor: 1 bytes, line 51
51
52<script type="text/javascript" src="/universal/scripts/yui2-yui3-bridge.js?CE=75"></script>
vendor: 1 bytes, line 52
52
53<script type="text/javascript" src="/universal/scripts/squarespace-gallery-slideshow.js?CE=75"></script>
53
54
55
56  
56<script type="text/javascript">
57  // <![CDATA[
58
59    Squarespace.Constants.SS_AUTHKEY = encodeURIComponent("");
60    Squarespace.Constants.WEBSITE_TITLE =  "thoughts from the red planet"; 
61    
62     Squarespace.Constants.CURRENT_MODULE_ID = "7220166"; 
63    
64    
65    
66    Squarespace.Constants.ADMINISTRATION_UI =  false; 
67    
68    
69    
70    Squarespace.Constants.CORE_APPLICATION_DOMAIN = "squarespace.com";
71    Squarespace.Constants.CORE_V6_APPLICATION_DOMAIN = "squarespace.com";
72    Squarespace.Constants.V5_FRONTSITE = "http://www.squarespace.com";
73    Squarespace.Constants.WEBSITE_ID = "621062";
74
75    
76
77    
78
79    
80
81  // ]]>
82  </script>
82
83
84      <title>thoughts from the red planet - thoughts from the red planet</title>
85    <meta property="og:title" content="thoughts from the red planet - thoughts from the red planet" /> 
86  
87
88  
89
90  
91
92  <!-- End Heading -->
93
94  <meta property="fb:admins" content="28109622" />
95<script type="text/javascript" src="/storage/syntax/shCore.js"></script>
vendor: 2 bytes, line 95
95
96<script type="text/javascript" src="/storage/syntax/shBrushClojure.js"></script>
96
97<link href="/storage/syntax/shCore.css" rel="stylesheet" type="text/css" />
98<link href="/storage/syntax/shThemeDefault.css" rel="stylesheet" type="text/css" />
99<link href="/storage/syntax/shClojureExtra.css" rel="stylesheet" type="text/css" />
100<style type="text/css">
101  .syntaxhighlighter { 
102     overflow-y: hidden !important; 
103     overflow-x: auto !important; 
104  }
105</style>
106<style type="text/css">
107
108/* ios safari line number fix */
109
110  /* iphone */
111  @media only screen and (max-device-width: 480px) {
112    td.gutter div.line { font-size: 1.45em !important; }
113  }
114
115</style>
116
117  
117<script type="text/javascript" src="/universal/scripts/cookie-min.js"></script>
117
118  
118<script type="text/javascript" src="/universal/scripts/eu-cookie-policy.js"></script>
118
119
120</head>
121
122<body  id="modulePage7220166"   >
123
124  
124<script type="text/javascript">
125  YAHOO.util.Event.onDOMReady(function() {
126    if (document.location.href.indexOf("SSLogoutOk=true") != -1) {
127      new Squarespace.FixedPositionTip("Logout Successful", "You have been successfully logged out.", { xMargin: 15, yMargin: 15, icon: "/universal/images/helptip-info.png", orientation: "upper-right", viewportFixed: true, autoHide: 1800 }).show();
128    }
129  });
130</script>
130
131
132    
133
134  
134<script type="text/javascript">
135    YAHOO.util.Event.onDOMReady(function() { new Squarespace.Census.hit(); } );
136  </script>
136
137
138  
138<script type="text/javascript">
139    YAHOO.util.Event.addListener(document, "keyup", function (e) { if (!e) { e = event; } if (e.keyCode == 27) { document.location.href = "https://nathanmarz.com/display/Login?returnUrl=%2Fblog%2F%3FcurrentPage%3D2"; } });
140  </script>
140
141
142
143
144
145  
146
147  <div id="canvasWrapper"><div id="canvas">
148  
149  
150
151  <div id="pageHeaderWrapper"><div id="pageHeader">
152
153      <div id="navigationTop"><div id="navigationTop-sectionContent3119044" class="horizontalNavigationBar"><ul class="content-navigation">
154    
155  <li class="module active-module  " id="navigationTop-moduleContentWrapper7220166">
156
157    <div id="navigationTop-moduleContent7220166">
158
159              <a id="navigationTop-moduleLink7220166" class=" " href="/" ><span>Blog</span></a>
160      
161
162      
163
164    </div>
165
166  </li>
167
168
169  <li class="module   " id="navigationTop-moduleContentWrapper7220167">
170
171    <div id="navigationTop-moduleContent7220167">
172
173              <a id="navigationTop-moduleLink7220167" class=" " href="/about/" ><span>About</span></a>
174      
175
176      
177
178    </div>
179
180  </li>
181
182
183  <li class="module   " id="navigationTop-moduleContentWrapper7282642">
184
185    <div id="navigationTop-moduleContent7282642">
186
187              <a id="navigationTop-moduleLink7282642" class=" " href="/archives/" ><span>Archives</span></a>
188      
189
190      
191
192    </div>
193
194  </li>
195
196
197  <li class="module   " id="navigationTop-moduleContentWrapper7223825">
198
199    <div id="navigationTop-moduleContent7223825">
200
201              <a id="navigationTop-moduleLink7223825" class=" " href="/contact/" ><span>Contact</span></a>
202      
203
204      
205
206    </div>
207
208  </li>
209
210
211  </ul><br class="clearer"/></div></div>
212
213    
214        
215      
216  
217      <div id="bannerAreaWrapper"><div id="bannerArea">
218
219              <div id="siteTitleWrapper">
220          <h1 id="siteTitle"><a href="https://nathanmarz.com/">thoughts from the red planet</a></h1>
221          
222        </div>
223      
224      
225      </div></div>
226    
227    
228    
229
230      <div id="navigationBottom"><div id="navigationBottom-sectionContent3119044" class="horizontalNavigationBar"><ul class="content-navigation">
231    
232  <li class="module active-module  " id="navigationBottom-moduleContentWrapper7220166">
233
234    <div id="navigationBottom-moduleContent7220166">
235
236              <a id="navigationBottom-moduleLink7220166" class=" " href="/" ><span>Blog</span></a>
237      
238
239      
240
241    </div>
242
243  </li>
244
245
246  <li class="module   " id="navigationBottom-moduleContentWrapper7220167">
247
248    <div id="navigationBottom-moduleContent7220167">
249
250              <a id="navigationBottom-moduleLink7220167" class=" " href="/about/" ><span>About</span></a>
251      
252
253      
254
255    </div>
256
257  </li>
258
259
260  <li class="module   " id="navigationBottom-moduleContentWrapper7282642">
261
262    <div id="navigationBottom-moduleContent7282642">
263
264              <a id="navigationBottom-moduleLink7282642" class=" " href="/archives/" ><span>Archives</span></a>
265      
266
267      
268
269    </div>
270
271  </li>
272
273
274  <li class="module   " id="navigationBottom-moduleContentWrapper7223825">
275
276    <div id="navigationBottom-moduleContent7223825">
277
278              <a id="navigationBottom-moduleLink7223825" class=" " href="/contact/" ><span>Contact</span></a>
279      
280
281      
282
283    </div>
284
285  </li>
286
287
288  </ul><br class="clearer"/></div></div>
289
290
291  </div></div>
292  
293  
294
295  
296
297  
298  <div id="pageBodyWrapper"><div id="pageBody">
299  
300  <div id="sidebar1Wrapper" class="verticalNavigationBarWrapper"><div class="iw-vnb1"><div class="iw-vnb2"><div class="iw-vnb3"><div class="iw-vnb4"><div id="sidebar1" class="verticalNavigationBar">
301
302  
303
304    <div id="sectionContent3119044" class="sectionWrapper promotedContainer "><div class="iw-s1"><div class="iw-s2"><div class="iw-s3"><div class="iw-s4"><div class="section">
305
306       <div class="caption">Top</div> 
307
308      
309        
310          <ul class="content-navigation">
311  <li class="module active-module  " id="moduleContentWrapper7220166">
312
313    <div id="moduleContent7220166">
314
315              <a id="moduleLink7220166" class=" " href="/" ><span>Blog</span></a>
316      
317
318      
319
320    </div>
321
322  </li>
323
324
325  <li class="module   " id="moduleContentWrapper7220167">
326
327    <div id="moduleContent7220167">
328
329              <a id="moduleLink7220167" class=" " href="/about/" ><span>About</span></a>
330      
331
332      
333
334    </div>
335
336  </li>
337
338
339  <li class="module   " id="moduleContentWrapper7282642">
340
341    <div id="moduleContent7282642">
342
343              <a id="moduleLink7282642" class=" " href="/archives/" ><span>Archives</span></a>
344      
345
346      
347
348    </div>
349
350  </li>
351
352
353  <li class="module   " id="moduleContentWrapper7223825">
354
355    <div id="moduleContent7223825">
356
357              <a id="moduleLink7223825" class=" " href="/contact/" ><span>Contact</span></a>
358      
359
360      
361
362    </div>
363
364  </li>
365
366</ul>
367
368        
369
370      
371  
372    </div></div></div></div></div></div>
373
374  
375
376    <div id="sectionContent3119896" class="sectionWrapper  "><div class="iw-s1"><div class="iw-s2"><div class="iw-s3"><div class="iw-s4"><div class="section">
377
378       <div class="caption">Follow Nathan on</div> 
379
380      
381        
382          <div class="content-passthrough"><div id="moduleContentWrapper7222502" class="widget-wrapper  widget-type-sociallinks"><div id="moduleContent7222502">
383
384
385
386  
387
388    
389    <ul class="social-link-list-pt">
390  
391      
392              <li id="socialLink98472" class="twitter">
393                <a  class="icon-link" style="background-image: url(/universal/images/social/twitter.png);"  onclick="this.target='new';" href="http://www.twitter.com/nathanmarz">Twitter</a>
394              </li>
395            
396              <li id="socialLink98474" class="">
397                <a  class="icon-link" style="background-image: url(http://nathanmarz.com/storage/github.png);"  onclick="this.target='new';" href="http://github.com/nathanmarz">GitHub</a>
398              </li>
399            
400              <li id="socialLink98494" class="">
401                <a  class="icon-link" style="background-image: url(http://nathanmarz.com/storage/rss_icon.gif);"  onclick="this.target='new';" href="http://feeds.feedburner.com/thoughtsfromtheredplanet?format=xml">Blog RSS</a>
402              </li>
403            
404              <li id="socialLink240093" class="">
405                <a  class="icon-link" style="background-image: url(http://nathanmarz.com/storage/envelope.png);"  onclick="this.target='new';" href="http://eepurl.com/b7x56z">Email subscribe</a>
406              </li>
407            
408  
409    </ul>
410    
411  
412
413</div></div><div id="moduleContentWrapper7223725" class="widget-wrapper  widget-type-page"><div id="moduleContent7223725">
414
415
416
417
418
419  <p class="sideTitle">Search</p>
420</div></div><div id="moduleContentWrapper7223723" class="widget-wrapper  widget-type-search"><div id="moduleContent7223723">
421
422
423<!-- <form method="get" action="/display/Search">
424	<div class="search-form-pt-wrapper">
425	  <div class="search-form-pt">
426	    <span class="queryFieldWrapper"><input type="text" class="text queryField" name="searchQuery" value=""/></span>
427	    <span class="queryButtonWrapper"><input type="submit" class="button queryButton" value="&raquo;"/></span>
428	    <input type="hidden" name="moduleId" value="7223723"/>
429	    <br class="clearer"/>
430	  </div>
431	</div>
432</form> -->
433
434<form method="get" action="/display/Search">
435	<input type="hidden" name="moduleId" value="7223723"/>
436	<table class="search-form-pt-wrapper search-form-table" border="0" cellpadding="0" cellspacing="0">
437		<tr class="search-form-pt search-form-row">
438			<td class="queryFieldCell search-form-cell">
439				<span class="queryFieldWrapper">
440					<input type="text" class="text queryField" name="searchQuery" value=""/>
441				</span>
442			</td>
443			<td class="queryButtonCell search-form-cell">
444				<span class="queryButtonWrapper">
445					<input type="submit" class="button queryButton" value="&rarr;"/>
446				</span>
447			</td>
448		</tr>
449	</table> 
450</form></div></div><div id="moduleContentWrapper7282531" class="widget-wrapper  widget-type-page"><div id="moduleContent7282531">
451
452
453
454
455
456  <p class="sideTitle">Featured Posts</p>
457  <ul class="sideList">
458  <li><a class="sideLink" href="http://nathanmarz.com/blog/suffering-oriented-programming.html">Suffering-oriented programming</a></li>
459  <li><a class="sideLink" href="http://nathanmarz.com/blog/history-of-apache-storm-and-lessons-learned.html">History of Apache Storm and lessons learned</a></li>
460  <li><a class="sideLink" href="http://nathanmarz.com/blog/principles-of-software-engineering-part-1.html">
460Principles of Software Engineering, Part 1</a></li>
461  <li><a class="sideLink" href="http://nathanmarz.com/blog/how-to-beat-the-cap-theorem.html">How to beat the CAP theorem</a></li>
462  <li><a class="sideLink" href="http://nathanmarz.com/blog/mimi-silbert-the-greatest-hacker-in-the-world.html">Mimi Silbert: the greatest hacker in the world</a></li>
463  <li><a class="sideLink" href="http://nathanmarz.com/blog/the-mathematics-behind-hadoop-based-systems.html">The mathematics behind Hadoop-based systems</a></li>
464  </ul>
465</div></div><div id="moduleContentWrapper7282699" class="widget-wrapper  widget-type-page"><div id="moduleContent7282699">
466
467
468
469
470
471  <p class="sideTitle">Latest Posts</p>
472</div></div><div id="moduleContentWrapper7282688" class="widget-wrapper  widget-type-journalarchive"><div id="moduleContent7282688">
473
474
475
476
477
478
479
480  <ul class="archive-item-list-pt">
481     <li><a href="/blog/why-i-walked-away-from-millions-of-dollars-to-found-a-startu.html">Why I walked away from millions of dollars to found a startup</a></li>  <li><a href="/blog/clojures-missing-piece.html">Clojure's missing piece</a></li>  <li><a href="/blog/recipe-for-a-great-programmer.html">Recipe for a great programmer</a></li>  <li><a href="/blog/how-becoming-a-pilot-made-me-a-better-programmer.html">How becoming a pilot made me a better programmer</a></li>  <li><a href="/blog/the-limited-value-of-a-computer-science-education.html">The limited value of a computer science education</a></li>  <li><a href="/blog/functional-navigational-programming-in-clojurescript-with-sp.html">Functional-navigational programming in Clojure(Script) with Specter</a></li>  <li><a href="/blog/history-of-apache-storm-and-lessons-learned.html">History of Apache Storm and lessons learned</a></li> 
482  </ul>
483
484
485
486</div></div><div id="moduleContentWrapper7223803" class="widget-wrapper  widget-type-page"><div id="moduleContent7223803">
487
488
489
490
491
492  <p class="sideTitle">All Posts</p>
493</div></div><div id="moduleContentWrapper7282709" class="widget-wrapper  widget-type-journalarchive"><div id="moduleContent7282709">
494
495
496
497
498
499
500
501
502
503  <ul class="archive-item-list-pt">
504     <li><a href="/blog/month/september-2021">September 2021</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/march-2017">March 2017</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/january-2017">January 2017</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/july-2016">July 2016</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/june-2016">June 2016</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/september-2015">September 2015</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/october-2014">October 2014</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/june-2014">June 2014</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/may-2014">May 2014</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/february-2014">February 2014</a> <span class="entry-count">(2)</span></li>  <li><a href="/blog/month/april-2013">April 2013</a> <span class="entry-count">(3)</span></li>  <li><a href="/blog/month/march-2013">March 2013</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/september-2012">September 2012</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/february-2012">February 2012</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/january-2012">January 2012</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/october-2011">October 2011</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/march-2011">March 2011</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/january-2011">January 2011</a> <span class="entry-count">(3)</span></li>  <li><a href="/blog/month/december-2010">December 2010</a> <span class="entry-count">(2)</span></li>  <li><a href="/blog/month/november-2010">November 2010</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/october-2010">October 2010</a> <span class="entry-count">(2)</span></li>  <li><a href="/blog/month/august-2010">August 2010</a> <span class="entry-count">(2)</span></li>  <li><a href="/blog/month/july-2010">July 2010</a> <span class="entry-count">(2)</span></li>  <li><a href="/blog/month/june-2010">June 2010</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/may-2010">May 2010</a> <span class="entry-count">(3)</span></li>  <li><a href="/blog/month/april-2010">April 2010</a> <span class="entry-count">(3)</span></li>  <li><a href="/blog/month/march-2010">March 2010</a> <span class="entry-count">(5)</span></li>  <li><a href="/blog/month/february-2010">February 2010</a> <span class="entry-count">(1)</span></li>  <li><a href="/blog/month/january-2010">January 2010</a> <span class="entry-count">(4)</span></li>  <li><a href="/blog/month/december-2009">December 2009</a> <span class="entry-count">(1)</span></li> 
505  </ul>
506
507</div></div></div>
508
509        
510
511      
512  
513    </div></div></div></div></div></div>
514
515  
516
517  
518
519</div></div></div></div></div></div>
520  <div id="sidebar2Wrapper" class="verticalNavigationBarWrapper"><div class="iw-vnb1"><div class="iw-vnb2"><div class="iw-vnb3"><div class="iw-vnb4"><div id="sidebar2" class="verticalNavigationBar">
521
522  
523
524  
525
526</div></div></div></div></div></div>
527  
528  
529
530  <div id="contentWrapper"><div id="content">
531    
532          
533    
534
535
536  
537
538
539  
540
541  
542    
543
544  
545  
546  <div class="list-journal-entry-wrapper">
547
548          
549
550
551<div class="journal-entry-wrapper post-text authored-by-nmarz ">
552<div id="item34627735" class="journal-entry">
553
554  <div class="journal-entry-float-day"><span class="name">Wednesday</span></div><div class="journal-entry-float-date"><span class="month">Feb</span><span class="date">12</span><span class="year">2014</span></div>
555  
556  
557  
558  <div class="journal-entry-text">
559  
560      
561      <h2 class="title">
562    
563        
564    
565        <a class="journal-entry-navigation-current" href="/blog/interview-with-programmer-magazine.html">Interview with &quot;Programmer&nbsp;Magazine&quot;</a>  
566     
567         
568    
569      </h2>
570      
571       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Wednesday, February 12, 2014</span> </div> 
572
573    
574  
575      
576      <div class="body">
577  
578        
579        
580        
581          <p><em>I was <a href="http://www.csdn.net/article/2014-02-05/2818311-Storm-Nathan-Marz">recently interviewed</a> for "Programmer Magazine", a Chinese magazine. The interview was published in Chinese, but a lot of people told me they'd like to see the English version of the interview. Due to the Google translation being, ahem, a little iffy, I decided to just publish the original English version on my blog. Hope you enjoy!</em></p>
582
583<h3>What drew you to programming and what was the first interesting program you wrote?</h3>
584
585<p>I started programming when I was 10 years old on my TI-82 graphing calculator. Initially I started programming because I wanted to make games on my calculator – and also because I was bored in math class :D. The first interesting game I made on my calculator was an archery game where you'd shoot arrows at moving targets. You'd get points for hitting more targets or completing all the targets faster. A couple years later I graduated to programming the TI-89 which was a huge upgrade in power. I remember how the TI-82 only let you have 26 variables (for the characters 'a' through 'z') and thinking how incredible it was that the TI-89 let you have as many variables as you want.</p>
586
587<h3>What do you do to improve your skills as a programmer?</h3>
588
589<p>I get better by doing a lot of programming and trying new things. One of the best ways to become a better programmer is to learn new programming languages. By learn I mean more than just learning the syntax of the language, I mean understanding the language's idioms and writing something substantial in 
589it. For me, learning Clojure made me a much better programmer in all languages.</p>
590
591<h3>Could you talk about your experience before joining BackType?</h3>
592
593<p>I got my bachelor's and master's in Computer Science at Stanford University with a focus on software theory. So I did a lot of algorithms and proofs and so on. Probably the best thing I did at Stanford was choose classes not so much by the subject material but by the professor. When I found a professor who was a great teacher I would take as many classes with that professor as possible. For example, one of the greatest teachers I've ever had is <a href="http://theory.stanford.edu/~tim/">Professor Tim Roughgarden</a>. I took a bunch of "algorithmic game theory" classes with him – algorithmic game theory is basically the intersection of economics and computer science. I took the classes not so much for the material but to improve my problem solving skills. Professor Roughgarden had an incredibly coherent and disciplined way of breaking down extremely difficult problems and making them easy to understand. Learning those skills has made me a much better problem solver in all scenarios, as well as being a much better communicator of difficult concepts.</p>
594
595<h3>You once said, leaving Twitter is a tough decision, could you please tell us why you decide start your own company? What object do you want to achieve?</h3>
596
597<p>I had a pretty great situation at Twitter, having my own team and working full-time on a project I started. But when I thought of the idea for my company, it was so compelling I just couldn't stop thinking about it. So I felt that if I didn't start this company, I would regret it for the rest of my life.  </p>
598
599<h3>What are the main lessons you learned in the last few years of your professional career?</h3>
600
601<p>Feedback is everything. Most of the time you're wrong, and feedback is the only way to realize your mistakes and help you become less wrong. This applies to everything. In product development, get your product out there as soon as possible so you can get feedback and see what works and what doesn't. In many cases you don't even need to build anything – a link to a "feature" that actually goes to a survey page can give you the feedback you need to test your idea. </p>
602
603<p>In managing a team, it's really important to have feedback on all the processes you do. At BackType we'd have a once a month meeting to discuss our processes and whether they are effecive or too restrictive. This caused us to introduce standups, and then remove standups when we didn't feel they were that useful to us. We used that process to go from monthly meetings to biweekly meetings to weekly meetings, then back to biweekly meetings. </p>
604
605<h3>In your blog you said "I'm always happy to give advice or connect with people doing interesting work", what interesting projects you have seen, and which suggestions you provide?</h3>
606
607<p>The founder of <a href="http://insightdatascience.com/">Insight Data Science</a> approached me when he was starting it, and I think it's an absolutely terrific program. They provide a 6 week bootcamp to help math/science/physics PhD's learn programming skills so they can start a career in data science. Basically the program recognizes that there is a surplus of very smart people who don't necessarily have the most interesting job prospects, while there is a booming tech industry with a huge talent shortage of data scientists. So they bridge that gap. I was able to help them out with a couple things and I think their execution has been very impressive.</p>
608
609<h3>What prompted you to write the book Big Data and what problems you want to solve? Writing a book will take a long process, what you have learned during this process?</h3>
610
611<p>I had developed a lot of theory and best practices about architecting big data systems that no one else was talking about. People were focused on very specific use cases, whereas I had developed rigorous, holistic approaches. A lot of the things I talk about, like being resilient to human error (something I consider to be absolutely non-negotiable) are ignored by the vast majority of industry. I think the industry will be much better off by building these systems more rigorously and less haphazardly, and I felt that this book was the right way to effect that change.</p>
612
613<p>I knew that writing a book would be a lot of work, but it turned about to be signicantly more work than I expected. I think my book is especially challenging because it's such a huge subject. At one point I had half the book written, but I realized I was taking the wrong approach to communicating the material so I scrapped everything and started over. It was definitely worth it though because based on the feedback I get from readers they love the material and really get what I'm trying to communicate. </p>
614
615<p>My editors have been absolutely invaluable in the writing process and have helped me become a much better writer. I've learned that the way I was taught in school to write is actually the complete opposite of effective communication. I was taught to make your general "thesis" statement up front, and then drill down into that general statement with supporting points and eventually specific details. It turns out that this forces the reader to do a lot of work to 
615synthesize everything you're saying. They won't grasp the thesis up front – because they haven't read the supporting points yet. So after drilling down the reader now has to drill back "up" to connect everything. It's a convoluted way to achieve an understanding of something. A much better way to communicate is to tell a story – start with a situation the reader already understands, and then connect step by step to the ultimate general statement you want your reader to understand. Specific to general is always better than general to specific.</p>
616
617<h3>You contributed a lot open source projects, what makes you believe in open source?</h3>
618
619<p>Open source benefits so many people in so many ways. When you're a startup, you're highly resource-constrained, so being able to take advantage of the work other people have done is a godsend. Lowering the cost of doing startups, of course, is highly beneficial to society. When you benefit that much from open source, you do feel obligated to give back as well. On top of that, when you open source software as a company, you benefit from other people trying it out, finding issues, and improving your software "for free". </p>
620
621<p>On a personal level, open source has given me an opportunity to interact with an entire world of developers, rather than just those in whatever company I happened to be at. This has been hugely beneficial to my career, allowing me to get to know tons of awesome people and travel the world to speak at conferences. </p>
622
623<h3>Which person has influenced you the most?</h3>
624
625<p>Philosophically I'd have to say the most influential person to me is Carl Sagan. I've read most of his books and find them hugely inspirational. I think he was one of the greatest communicators of all time, and what impresses me most about him is his extreme empathy towards his audience. For example, he has quite a bit of writing about science vs. religion – but as a scientist he is not hostile towards religion or anything like that. He understands why people are religious and the value they get out of it. So when he communicates the value of science and skepticism to religious people he starts with religion = valuable as a starting point. That degree of empathy is really rare, and it's something that I'm continuously trying to improve at. He taught me that empathy is the basis of good communication.</p>
626
627<h3>What's the key points John McCarthy told you about his life and perspective? How these words affect the rest of your life, what is your own life and perspective?</h3>
628
629<p>I talked with John McCarthy for two hours when I was a sophomore in college. The most striking thing he told me was when I asked about the history of Lisp. He told me he needed a better programming language for doing AI research, so he invented Lisp for that purpose. He really didn't seem to care that much about programming languages – his real passion was AI. It struck me as exactly like how Isaac Newton invented calculus because he needed it for his physics work. The pebbles of giants really are big boulders. </p>
630
631<h3>When designing a software system, which process you will use? (the fist step, the second step...)</h3>
632
633<p>I think designing a software system is entirely about learning what to build as you go. I use a technique which I call "suffering-oriented programming" in order to maximize learning and minimize wasted work. I detailed this approach <a href="http://nathanmarz.com/blog/suffering-oriented-programming.html">on my blog</a>. The general idea is to avoid making "general" or "extensible" solutions until you have a very deep understanding of the problem domain. Instead you should hack things out very directly to get a working prototype as fast as possible. Then you iterate and evolve and learn more about the problem domain. Once you have a good understanding of the intricacies of the problem domain, then you can redesign your solution to make it more general, extensible, etc. Finally, at the end, you wrap things up by tightening up the code and making performance optimizations. The sequence is "First make it possible. Then make it beautiful. Then make it fast."</p>
634
635<h3>Do you have any principle when you programming?</h3>
636
637<p>I believe strongly in immutability and referentially transparent functions as ways to vastly simplify software. Mutability creates a web of dependencies in your code – things that can change other things which then change other things – which gets hard to wrap your head around. Code is all about being able to understand what's going on, so anything you do to make that easier is a good thing. Immutability is one such technique to reduce what you need to understand about a particular piece of code to grasp it. Additionally, referentially transparent functions only depen
637d on their arguments (and not on any other state), so they are also easier to understand. </p>
638
639<p>Another important principle I live by is "my code is wrong". I think it's pretty clear that we don't know how to make perfect software – all the code I've ever used or written has had bugs in it. So I assume my code is wrong and design it to work anyway (with higher probability, at least). I've detailed techniques to accomplish this on my blog and in my conference talks this year.</p>
640
641<h3>Compared with your early years, what is the biggest change when you programming today?</h3>
642
643<p>Since I started off programming graphing calculators, I'd say the biggest change is using full-fledged keyboards to program instead of those tiny keypads :)</p>
644
645<h3>Storm development is in a very short time, by few developers, under a limited budget and urgent requirements, what is your secret about how could be so efficient like this?</h3>
646
647<p>Storm is the result of following that "suffering-oriented programming" methodology. We didn't jump into Storm out of the blue – we had been doing realtime computation at BackType for a long time by stringing workers together manually with queues. So we had a really solid understanding of our needs for realtime processing. Storm at its core is just a simple set of abstractions with a clever algorithm behind the scenes to guarantee data processing. When I thought long and hard about the realtime problems we were dealing with at the time, the design of Storm was obvious. Additionally, I had a ton of experience with Hadoop and knew some of the mistakes made in the design of that system, so I applied that learning into Storm to make it more robust. </p>
648
649<h3>You have interviewed a lot of programmers, what are the best programmers in common do you think?</h3>
650
651<p>The best programmers are obsessed with improving as programmers. They love exploring new programming languages and new ideas. Another key trait of great programmers is a "getting stuff done" mentality. It's far more important to get something working than to make the perfect design. Plus a great programmer recognizes that you can't make a perfect design without first having something working that you can learn from.</p>
652
653<h3>Are there any myths (layman think something right, but expert do not think so) and traps in Data System and Big Data?</h3>
654
655<p>Probably the biggest misconception I see is people placing the relational database, and associate concepts like CRUD, on a pedestal. People treat the RDBMS as if its the ultimate in database technology, and everyone seems to be trying to recreate the RDBMS to work in Big Data land. But this ignores massive problems that have always existed with the RDBMS: they're based on mutability so are extremely susceptible to corruption whenever there's a bug or a human error, and they force you into a horrible situation of needing to either normalize your schema and take performance hits, or denormalize your schema and create a maintenance nightmare (among other problems). When you actually look at data systems from first principles, as I do in my book, you see that there's different ways of architecting data systems that have none of these complexities.</p>
656
657<h3>What problem you want to solve in BackType that lead you decide to start design Storm?</h3>
658
659<p>There were two problems. The first was how to keep our databases containing social media analytics stats up to date in realtime in a reliable way. The second was the "reach problem" – how to compute the "reach" of a URL on Twitter very quickly. The "reach" is the unique count of all the followers of all the people who tweeted a URL. It's very computationally intensive and hard to precompute. Storm turned out to be a simple abstraction which unified these seemingly unrelated use cases.</p>
660
661<h3>What reason or experience make you sure that you will successfully build Storm?</h3>
662
663<p>The key was that we had tons of experience with realtime computation so knew the problem domain very well. So there was really no question in my mind that Storm would be successful because I had already learned a majority of the little gotchas.</p>
664
665<h3>Why you choose Clojure as the development language of Storm? Could you talk about your long practical experience about using this language (like its advantages and disadvantages)? Which feature won't appear in the Storm, if you were not using Clojure?</h3>
666
667<p>Clojure is the best language I've ever used, by far. I use it because it makes me vastly more productive by allowing me to easily use techniques like immutability and functional programming. Its dynamic nature by being Lisp-based ensures that I can always mold Clojure as necessary to formulate the best possible abstractions. Storm would not be any different if I didn't use Clojure, it just would have been far more painful to build.</p>
668
669<h3>
669From your blog, I saw you advocate a lot about writing. Could you share us what do you do in improve your writing skill?</h3>
670
671<p>The only way to improve at writing is to write a lot. When other people read my writing and give me feedback, like by commenting on my blog, I carefully think about where that comment came from. If they misunderstood something then that means I'm not communicating correctly – either I'm not clear or I'm not properly anticipating reader objections (whether or not those objections are fallacious is irrelevant). By understanding why my message doesn't get through, I'm able to do a better job the next time.</p>
672
673<p>I also read a lot and try to learn from great writers. As I've mentioned Carl Sagan is one of my favorite writers and I've learned tons from reading him – and I continue to learn tons from him everytime I read his work.</p>
674
675<h3>You start using Emacs recent years, could you talk about the programming tools you choose and how they impact you?</h3>
676
677<p>I started using Emacs because I found it to be the best environment for programming Clojure (due to its Lisp background). I've been really impressed with how powerful of a tool it is and how much it can be customized to my needs. On top of that, since it was originally written so long ago, it has an incredibly small resource footprint. That's something I really enjoy because modern IDE's tend to be such resource hogs. </p>
678
679<p>Other than that, I think my setup is pretty simplistic. I use a live REPL in my Emacs for exploratory development and interactive testing. I also have tons of text files on my computer with design notes and ideas. For my todo list I literally just use a text file.</p>
680
681
682        
683  
684          
685  
686         
687  
688      </div>
689
690    
691
692  </div>
693
694      <div class="journal-entry-tag journal-entry-tag-post-body">
695      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/interview-with-programmer-magazine.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Finterview-with-programmer-magazine.html','Interview%20with%20%22Programmer%20Magazine%22');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
696      <div class="journal-entry-tag-post-body-line2"></div>
697      <div class="journal-entry-tag-post-body-line3"></div>
698    </div>
699  
700  
701  <div class="clearer"></div>
702
703</div>
704</div>
705
706
707
708
709
710
711
712<div class="journal-entry-wrapper post-text authored-by-nmarz ">
713<div id="item33320532" class="journal-entry">
714
715  <div class="journal-entry-float-day"><span class="name">Friday</span></div><div class="journal-entry-float-date"><span class="month">Apr</span><span class="date">12</span><span class="year">2013</span></div>
716  
717  
718  
719  <div class="journal-entry-text">
720  
721      
722      <h2 class="title">
723    
724        
725    
726        <a class="journal-entry-navigation-current" href="/blog/break-into-silicon-valley-with-a-blog-1.html">Break into Silicon Valley with a&nbsp;blog</a>  
727     
728         
729    
730      </h2>
731      
732       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Friday, April 12, 2013</span> </div> 
733
734    
735  
736      
737      <div class="body">
738  
739        
740        
741        
742          <p>I know a lot of non-technical people who would love to work in the venture-funded startup world, from consultants to finance people to other business types for which I'm not really sure exactly what it is they do. They hit obstacles trying to get into the startup world, finding that their skills are either irrelevant or hard to explain. My advice to all these people is the same:</p>
743
744<p><strong>Write a blog.</strong></p>
745
746<p>A blog can improve your life in enormous ways. Or to put it in business-s
746peak: a blog has one of the highest ROI's of anything you can do.</p>
747
748<p>Put yourself in the shoes of startups looking for talent. First off – startups are <em>desperate</em> for talent. The problem is that it's very difficult to identify great people – startups search through loads and loads of candidates.</p>
749
750<p>Resumes and interviews only tell you so much about a person. It's really hard to stand out in a resume – you're not the only one putting over-inflated impressive-looking numbers and bullet points on your resume. And interviews are notorious for labeling bad people as good and good people as bad. So to maximize your odds of making it through the funnel, you need to show that you're awesome independent from the randomness of the normal process.</p>
751
752<p>One thing you can do is write an insightful blog. This makes you look a lot more compelling. Now the reaction from startups will be "Hey, this person's really smart. We don't want to miss out on a potentially great hire, so let's put in a lot of effort to determine if there's a good fit." </p>
753
754<h3>A new dimension of opportunity</h3>
755
756<p>There's another <strong>huge</strong> advantage to having a blog besides being a mechanism to show that you're smart and insightful. A blog opens up a whole new dimension of opportunity for you. Instead of relying purely on outbound opportunities that you explicitly seek out yourself, you also will get inbound opportunities where people reach out <strong>to you</strong> with opportunities you never expected or dreamed of.</p>
757
758<p>With an outbound opportunity you know exactly what you're seeking, whether it's landing a job or speaking at a conference or something else. Inbound opportunities, on the other hand, are highly uncertain. They come to you out of the blue. In my personal experience, many of the most awesome things I've done started as inbound opportunities: a <a href="http://manning.com/marz">book deal</a>, flying all around the world for free to speak at conferences, <a href="http://www.youtube.com/watch?v=Ipjrhue5bXs">a keynote</a> at a major conference, and connecting with hundreds of awesome people who have reached out to me because of something I did or wrote publicly. </p>
759
760<p>When you write a blog, you greatly increase the likelihood of getting awesome inbound opportunities. When it comes to breaking into Silicon Valley – instead of everything being on your shoulders to seek interesting companies, those companies will be reaching out to you.</p>
761
762<p>A great phrase I've heard for this is "<a href="http://www.codusoperandi.com/posts/increasing-your-luck-surface-area">increasing your luck surface area</a>". By providing value to people publicly, like writing insightful posts on a blog, you open yourself up to serendipitous, "lucky" opportunities.</p>
763
764<h3>Getting readers</h3>
765
766<p>Besides writing smart posts, you also need people to read your writing. Here's a few tips for accomplishing that.</p>
767
768<p>First off, the title of a blog post is incredibly important. In very few words, you need to sell your potential reader that your blog post is going to be worth their time. I've found the the best titles are relevant to the potential reader, somewhat mysterious, and non-generic. Titles are definitely an art form, so you should think hard about how you'll name your posts. Sometimes I wait days to publish a post because I haven't thought of a compelling enough title.</p>
769
770<p>Second, I highly recommend using Twitter as a distribution platform for your blog posts. The combination of Twitter and blogging leads to a beautifully virtuous cycle: your blog increases your Twitter following, and as your Twitter following grows you increase the reach of your blog. I consider Twitter to be the greatest professional networking tool ever devised – I follow people who tweet/blog interesting things and they follow me for the same reason. Then when I go to conferences I seek out the people who I know and respect from their online presence. When we meet, we already know a lot about each other and have a lot to talk about.</p>
771
772<p>Lastly, you should embrace the online communities who will care about your blog. In Silicon Valley, the most important community is <a href="https://news.ycombinator.com/news">Hacker News</a>. Hacker News is widely read in Silicon Valley by programmers, entrepreneurs, and investors. It c
772an drive a lot of readers to your blog in a short amount of time.</p>
773
774<p>Initially, it may be hard for you to get readers. Getting your posts on Hacker News is very much a crapshoot, and initially you'll have too small of a Twitter following to get that much distribution. But occasionally you'll write something smart that gets on Hacker News and gets shared around. Over time as your writing and distribution improves getting readers gets easier and easier.</p>
775
776<h3>What to write about</h3>
777
778<p>If you don't think you have anything to write about, then let me ask you a question. Do you really have that low of an opinion of yourself? Do you really think you have nothing interesting that you can share with the world? There's tons of stuff that you can write about that you don't even know to share. You have a ton of knowledge that you don't realize other people don't know because you spend all your time in your own head. Tell stories of times that you hustled. Write about the dynamics of big companies. Write case studies of anything related to running a business. Analyze the market for interesting new technologies (e.g. 3D printing, Bitcoin, etc). There's so much that you can write about.</p>
779
780<p>Once you start blogging, you'll become attuned to random ideas you have throughout the day that would make good blog posts. Most of my blog ideas start off as email reminders to myself. </p>
781
782<h3>Final thoughts</h3>
783
784<p>If you haven't blogged before, you're going to suck at first. Being accurate, precise, and insightful is not enough. You have to learn how to hook people into your posts and keep the post engaging. You'll learn about the glorious world of internet commenting, where people constantly misinterpret what you say and apply very fallacious reasoning to your posts. You'll see people trash your ideas on Hacker News even though it's clear they didn't read your entire post. Sometimes they comment having only read the title! You'll learn over time different ways to structure the same information in order to minimize misinterpretation. You'll learn to anticipate fallacious reasoning and preemptively address those fallacies.</p>
785
786<p>With writing, practice most definitely makes perfect. I sucked at writing at first, but I quickly improved.</p>
787
788<p>A lot of people say they "don't have time to write." To be blunt, I think comments like this are the result of laziness and self-delusion. Writing a blog is really not that much work. You really can't find a couple hours to pump out a blog post? Just occasionally, instead of going out to the bar or seeing a movie or going surfing or doing whatever it is you do for fun, try writing. The potential benefits relative to the investment are <strong>MASSIVE</strong>. I haven't even discussed all the <a href="http://nathanmarz.com/blog/you-should-blog-even-if-you-have-no-readers.html">other benefits</a> which on their own make blogging worthwhile.</p>
789
790<p>Of course, writing isn't the only thing you can do to help yourself break into Silicon Valley. But it's an enormously easy way to make yourself stand out and open yourself to opportunities you never expected.</p>
791
792<p><em>You should follow me on Twitter <a href="https://twitter.com/nathanmarz">here</a>.</em></p>
793
794
795        
796  
797          
798  
799         
800  
801      </div>
802
803    
804
805  </div>
806
807      <div class="journal-entry-tag journal-entry-tag-post-body">
808      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/break-into-silicon-valley-with-a-blog-1.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fbreak-into-silicon-valley-with-a-blog-1.html','Break%20into%20Silicon%20Valley%20with%20a%20blog');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
809      <div class="journal-entry-tag-post-body-line2"></div>
810      <div class="journal-entry-tag-post-body-line3"></div>
811    </div>
812  
813  
814  <div class="clearer"></div>
815
816</div>
817</div>
818
819
820
821
822
823
824
825<div class="journal-entry-wrapper post-text authored-by-nmarz ">
826<div id="item33177519" class="journal-entry">
827
828  <div class="journal-entry-float-day"><span class="name">Tuesday</span></div><div class="journal-entry-float-date"><span class="month">Apr</span><span class="date">02</span><span class="year">2013</span></div>
829  
830  
831  
832  <div class="journal-entry-text">
833  
834      
835      <h2 class="title">
836    
837        
838    
839        <a class="journal-entry-navigation-current" href="/blog/principles-of-software-engineering-part-1.html">
839Principles of Software Engineering, Part&nbsp;1</a>  
840     
841         
842    
843      </h2>
844      
845       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Tuesday, April 2, 2013</span> </div> 
846
847    
848  
849      
850      <div class="body">
851  
852        
853        
854        
855          <p>This is the first in a series of posts on the principles of software engineering. There's far more to software engineering than just "making computers do stuff" – while that phrase is accurate, it does not come close to describing what's involved in making robust, reliable software. I will use my experience building large scale systems to inform a first principles approach to defining what it is we do – or should be doing – as software engineers. I'm not interested in tired debates like dynamic vs. static languages – instead, I intend to explore the really core aspects of software engineering. </p>
856
857<p>The first order of business is to define what software engineering even is in the first place. Software engineering is the construction of software that produces some desired output for some range of inputs. The inputs to software are more than just method parameters: they include the hardware on which it's running, the rate at which it receives data, and anything else that influences the operation of the software. Likewise, the output of software is more than just the data it emits and includes performance metrics like latency.</p>
858
859<p>I think there's a distinction between programming a computer and software engineering. Programming is a deterministic undertaking: I give a computer a set of instructions and it executes those instructions. Software engineering is different. One of the most important realizations I've had is that while software is deterministic, you can't treat it as deterministic in any sort of practical sense if you want to build robust software. </p>
860
861<p>Here's an anectode that, while simple, hits on a lot of what software engineering is really about. At Twitter my team operated a <a href="http://storm-project.net/">Storm</a> cluster used by many teams throughout the company for production workloads. Storm depends on <a href="http://zookeeper.apache.org/">Zookeeper</a> to store various pieces of state relating to Storm's operation. One of the pieces of state stored is information about recent errors in application workers. This information feeds a Storm UI which users look at to see if their applications have any errors in them (the UI also showed other things such as statistics of running applications). Whenever an error bubbles out of application code in a worker, Storm automatically reports that error into Zookeeper. If a user is suppressing the error in their application code, they can call a "reportError" method to manually add that error information into Zookeeper. </p>
862
863<p>There was a serious omission in this design: we did not properly consider how that reportError method might be abused. One day we suddenly received a flood of alerts for the Storm cluster. The cluster was having serious problems and no one's application was running properly. Workers were constantly crashing and restarting.</p>
864
865<p>All the errors were Zookeeper related. I looked at the metrics for Zookeeper and saw it was completely overloaded with traffic. It was very strange and I didn't know what could possibly be overloading it like that. I took a look at which Zookeeper nodes were receiving the most API calls, and it turned out almost all the traffic was coming to the set of nodes used to store errors for one particular application running on the cluster. I shut that application down and the cluster immediately went back to normal.</p>
866
867<p>The question now was why that application was reporting so many errors. I took a closer look at the application and discovered that all the errors being reported were null pointer exceptions – a user had submitted an application with a bug in it causing it to throw that exception for every input tuple. In addition, the application was catching every exception, suppressing it, and manually calling reportError. This was causing reportError to be called at the same rate at which tuples were being received – which was a lot.</p>
868
869<p>An unfortunate interaction between two mistakes led to a major failure of a production system. First, a user deployed buggy, sloppy code to the cluster. Second, the reportError method had an assumption in it that errors were rare and thereby the amount of traffic to that method would be inconsequential. The user's buggy code broke that assumption, overloading Zookeeper and causing a cascading failure that took down every other application on the cluster. We fixed the problem by throttling the rate at which errors could be reported to Zookeeper: errors reported beyond that rate would be logged locally but not written to Zookeeper. This made reportError robust to high traffic and eliminated the possibility for cascading failure due to abuse of that functionality.</p>
870
871<p>
871As this story illustrates, there's a lot of uncertainty in software engineering. You think your code is correct – yet it still has bugs in it. How your software is actually used differs from the model in your head when you wrote the code. You made all sorts of assumptions while writing the software, some of which are broken in practice. Your dependencies, which you use as a black box, randomly fail due to a misunderstanding of their functional input range. The most salient feature of software engineering is the degree to which uncertainty permeates every aspect of the construction of software, from designing it to implementing it to operating it in a production environment. </p>
872
873<h3>Learning from other fields of engineering</h3>
874
875<p>It's useful to look at other forms of engineering to learn more about software engineering. Take bridge engineering, for example. The output of a bridge is a stable platform for crossing a chasm. Even though a bridge is a static structure, there are many inputs: the weight of the vehicles crossing, wind, rain, snow, the occasional earthquake, and so on. A bridge is engineered to operate correctly under certain ranges of those inputs. There's always some magnitude of an earthquake for which a bridge will not survive, and that's deemed okay because that's such a low probability event. Likewise, most bridges won't survive being hit by a missile.</p>
876
877<p>Software is similar. Software operates correctly only within a certain range of inputs. Outside those inputs, it won't operate correctly, whether it's failure, security holes, or just poor performance. In my Zookeeper example, the Zookeeper cluster was hit with more traffic than it could handle, leading to application failure. Similarly, a distributed database can only handle so many hardware failures in a short amount of time before failing in some respect, like losing data. That's fine though, because you tune the replication factor until the probability of such an event is low enough.</p>
878
879<p>Another useful field to look at is rocket engineering. It takes a lot of failure and iteration to build a rocket that works. SpaceX, for example, had three failed rocket launch attempts before they finally reached orbit. The cause of failure was always something unexpected, some series of inputs that the engineers didn't account for. Rockets are filled to the brim with telemetry so that failures can be understood, learned from, and fixed. Each failure lets the engineers understand the input ranges to the rocket a little better and better engineer the rocket to handle a greater and greater part of the input space. A rocket is never finished – you never know when there will be some low probability series of inputs you haven't experienced yet that will lead to failure. <a href="http://en.wikipedia.org/wiki/Space_Shuttle_Columbia_disaster">STS-107</a> was the 113th launch of the Space Shuttle, yet it ended in disaster.</p>
880
881<p>Software is very similar. Making software robust is an iterative process: you build and test it as best you can, but inevitably in production you'll discover new areas of the input space that lead to failure. Like rockets, it's crucial to have excellent monitoring in place so that these issues can be diagnosed. Over time, the uncertainty in the input space goes down, and software gets "hardened". SQL injection attacks and viruses are great examples of things that take advantage of software that operates incorrectly for part of its input space.</p>
882
883<div align=center>
884<span class="full-image-block ssNonEditable"><span><img src="/storage/input-space-venn.png?__SQUARESPACE_CACHEVERSION=1364803960668" style="width: 80%" alt=""/></span></span>
885</div>
886
887<p>There's always going to be some part of the input space for which software fails – as an engineer you have to balance the probabilities and cost tradeoffs to determine where to draw that line. For all of your dependencies, you better understand the input ranges for which the dependencies operate within spec and design your software accordingly.</p>
888
889<h3>Sources of uncertainty in software</h3>
890
891<p>There are many sources of uncertainty in software. The biggest is that we just don't know how to make perfect software: bugs can and will be deployed to production. No matter how much testing you do, bugs will slip through. Because of this fact of software development, all software must be viewed as probabilistic. The code you write only has some probability of being correct for all inputs. Sometimes seemingly minor failures will interact in ways that lead to much greater failures like in my Zookeeper example.</p>
892
893<p>Another source of uncertainty is the fact that humans are involved in running software in production. Humans make mistakes – almost every software engineer has accidentally deleted data from a database at some point. I've also experienced many episodes where an engineer accidentally launched a program that overloaded a critical internal service, leading to cascading failures. </p>
894
895<p>Another source of uncertainty is what functionality your software should even have – very rarely are the specs fully understood and fleshed out from the get go. Instead you have to learn as you go, iterate, and refine. This has huge implications on how software should be constructed and creates tension between the desire to create reusable components and the desire to avoid wasted work. </p>
896
897<p>There's uncertainty in all the dependencies you're using. Your dependencies will have bugs in them or will have unex
897pected behavior for certain inputs. The first time I hit a file handle limit error on Linux is an example of not understanding the limits of a dependency. </p>
898
899<p>Finally, another big source of uncertainty is not understanding the range of inputs your software will see in production. This leads to anything from incorrect functionality to poor performance to security holes like injection or denial of service attacks. </p>
900
901<p>This is by no means an exhaustive overview of sources of uncertainty in software, but it's clear that uncertainty permeates all of the software engineering process.</p>
902
903<h3>Engineering for uncertainty</h3>
904
905<p>You can do a much better job building robust software by being cognizant of the uncertain nature of software. I've learned many techniques over the years on how to design software better given the inherent uncertainties. I think these techniques should be part of the bread and butter skills for any software engineer, but I think too many engineers fall under the "software is deterministic" reasoning trap and fail to account for the implications of unexpected events happening in production. </p>
906
907<h4>Minimize dependencies</h4>
908
909<p>One technique for making software more robust is to minimize what your software depends on – the less that can go wrong, the less that will go wrong. Minimizing dependencies is more nuanced than just not depending on System X or System Y, but also includes minimizing dependencies on features of systems you are using.</p>
910
911<p>Storm's usage of Zookeeper is a good example of this. The location of all workers throughout the cluster is stored in Zookeeper. When a worker gets reassigned, other workers must discover the new location as quickly as possible so that they can send messages to the correct place. There are two ways for workers to do this discovery, either via the pull method or the push method. In the pull method, workers periodically poll Zookeeper to get the updated worker locations. In the push method, a Zookeeper feature called "watches" is used for Zookeeper to send the information to all workers whenever the locations change. The push method immediately propogates the information, making it faster than the pull method, but it introduces a dependency on another feature of Zookeeper. </p>
912
913<p>Storm uses <strong>both</strong> methods to propogate the worker location information. Every few seconds, Storm polls for updated worker information. In addition to this, Storm uses Zookeeper watches as an <strong>optimization</strong> to try to get the location information as fast as possible. This design ensures that even if the Zookeeper watch feature fails to work, a worker will still get the correct location information (albeit a bit slower in that particular instance). So Storm is able to take advantage of the watch feature without being fundamentally dependent on it. Most of the time the watch feature will work correctly and information will propogate quickly, but in the case that watches fail Storm will still work. This design turned out to be farsighted, as there was <a href="https://issues.apache.org/jira/browse/ZOOKEEPER-961">a serious bug</a> in watches that would have affected Storm.</p>
914
915<p>There's always a tradeoff between minimizing dependencies and minimizing the amount of code you need to produce to implement your application. In this case, doing the dual approach to location propogation was a good approach because it was a very small amount of code to achieve independence from that feature. On the other hand, removing Zookeeper as a dependency completely would not have been a good idea, as replicating that functionality would have been a huge amount of work and less reliable than just using a widely-used open-source project. </p>
916
917<h4>Lessen probability of cascading failures</h4>
918
919<p>A cascading failure is one of the worst things that can happen in production – when it happens it feels like the world is falling apart. One of the most common causes of cascading failures in my experience are accidental denial of service attacks like in my reportError example. The ultimate cause in these cases is a failure to respect the functional input range for components in your system. You can greatly reduce cascading failures by making interactions between components in your system explicitly respect those input ranges by using self
919-throttling to avoid accidental DOS'ng. This is the approach I used in my reportError example.</p>
920
921<p>Another great technique for avoiding cascading failures is to isolate your components as much as possible and take away the ability for different components to affect each other. This is often easier said than done, but when possible it is a very useful technique.</p>
922
923<h4>Measure and monitor</h4>
924
925<p>When something unexpected happens in production, it's critical to have thorough monitoring in place so that you can figure out what happened. As software hardens more and more, unexpected events will get more and more infrequent and reproducing those events will become harder and harder. So when one of those unexpected events happens, you want as much data about the event as possible.</p>
926
927<p>Software should be designed from the start to be monitored. I consider the monitoring aspects of software just as important as the functionality of the software itself. And everything should be measured – latencies, throughput stats, buffer sizes, and anything else relevant to the application. Monitoring is the most important defense against software's inherent uncertainty.</p>
928
929<p>In the same vein, it's important to do measurements of all your components to gain an understanding of their functional input ranges. What throughputs can each component handle? How is latency affected by more traffic? How can you break those components? Doing this measurement work isn't glamorous but is essential to solid engineering.</p>
930
931<h3>Conclusion</h3>
932
933<p>Software engineering is a constant battle against uncertainty – uncertainty about your specs, uncertainty about your implementation, uncertainty about your dependencies, and uncertainty about your inputs. Recognizing and planning for these uncertainties will make your software more reliable – and make you a better engineer. </p>
934
935<p><em>You should follow me on Twitter <a href="https://twitter.com/nathanmarz">here</a>.</em></p>
936
937
938        
939  
940          
941  
942         
943  
944      </div>
945
946    
947
948  </div>
949
950      <div class="journal-entry-tag journal-entry-tag-post-body">
951      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/principles-of-software-engineering-part-1.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />1 Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fprinciples-of-software-engineering-part-1.html','Principles%20of%20Software%20Engineering%2C%20Part%201');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
952      <div class="journal-entry-tag-post-body-line2"></div>
953      <div class="journal-entry-tag-post-body-line3"></div>
954    </div>
955  
956  
957  <div class="clearer"></div>
958
959</div>
960</div>
961
962
963
964
965
966
967
968<div class="journal-entry-wrapper post-text authored-by-nmarz ">
969<div id="item33179309" class="journal-entry">
970
971  <div class="journal-entry-float-day"><span class="name">Monday</span></div><div class="journal-entry-float-date"><span class="month">Apr</span><span class="date">01</span><span class="year">2013</span></div>
972  
973  
974  
975  <div class="journal-entry-text">
976  
977      
978      <h2 class="title">
979    
980        
981    
982        <a class="journal-entry-navigation-current" href="/blog/my-new-startup.html">My new&nbsp;startup</a>  
983     
984         
985    
986      </h2>
987      
988       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Monday, April 1, 2013</span> </div> 
989
990    
991  
992      
993      <div class="body">
994  
995        
996        
997        
998          <p>There's been a lot of speculation about what my new startup is doing, so I've decided to set the record straight and reveal all. We are working on one of the biggest problems on Earth, a problem that affects nearly every person on this planet. Our products will significantly improve the quality of life for billions of people.</p>
999
1000<p>We are going to revolutionize the bedsheet industry.</p>
1001
1002<div align=center>
1003<span class="full-image-block ssNonEditable"><span><img style="width: 80%" src="/storage/bullshit-sheets.png?__SQUARESPACE_CACHEVERSION=1364841735072" alt=""/></span></span>
1004</div>
1005
1006<p>Think about it. There's been almost no innovation in bedsheets in thous
1006ands of years. There's nothing worse than waking up to discover one of the corners of your Egyptian cotton fitted sheets has slipped off the mattress. How is this not a solved problem yet? Why are we still using sheets with that annoying elastic in it to secure them to our mattresses? They slip all the time – and if you have a deep mattress, good luck finding sheets that even fit. You're just screwed.</p>
1007
1008<p>Consider the impact of solving this problem, of a bedsheet product that <strong>never</strong> slips, that <strong>always</strong> stays secure on the mattress. This translates to better sleep, to less grogginess in the morning, to feeling more upbeat in the morning. This translates into less morning arguments between husbands and wives that spiral into divorces, child custody battles, and decades of trauma for the children. </p>
1009
1010<p>Not only is this a big problem – it's a big opportunity. We've done extensive market research and discovered that our target market is the entire human population. At 7 billion people and an estimated average sale of $20 per sheet, this is at least a $140,000,000,000 opportunity.</p>
1011
1012<p>We are going to solve this problem using modern, 21st century techniques and take bedsheets out of the Stone Age and into the future. We are going to make it possible to <em>attach your sheets to your bed</em>, completely solving the problem.</p>
1013
1014<p>Solving this problem in a practical and cost-efficient way is not easy and will require significant engineering breakthroughs. If you're a world-class, rock star by day and ninja by night engineer who's as passionate about bedsheets as I am, please get in touch. I'd love to talk to you.</p>
1015
1016<p>Bedsheets have been my true passion since I was a child. I'm excited to finally be focused on what I really care about, and I can't wait until the day when untucked sheets are a curious relic of the past.</p>
1017
1018
1019        
1020  
1021          
1022  
1023         
1024  
1025      </div>
1026
1027    
1028
1029  </div>
1030
1031      <div class="journal-entry-tag journal-entry-tag-post-body">
1032      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/my-new-startup.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fmy-new-startup.html','My%20new%20startup');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1033      <div class="journal-entry-tag-post-body-line2"></div>
1034      <div class="journal-entry-tag-post-body-line3"></div>
1035    </div>
1036  
1037  
1038  <div class="clearer"></div>
1039
1040</div>
1041</div>
1042
1043
1044
1045
1046
1047
1048
1049<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1050<div id="item33051507" class="journal-entry">
1051
1052  <div class="journal-entry-float-day"><span class="name">Saturday</span></div><div class="journal-entry-float-date"><span class="month">Mar</span><span class="date">16</span><span class="year">2013</span></div>
1053  
1054  
1055  
1056  <div class="journal-entry-text">
1057  
1058      
1059      <h2 class="title">
1060    
1061        
1062    
1063        <a class="journal-entry-navigation-current" href="/blog/leaving-twitter.html">Leaving&nbsp;Twitter</a>  
1064     
1065         
1066    
1067      </h2>
1068      
1069       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Saturday, March 16, 2013</span> </div> 
1070
1071    
1072  
1073      
1074      <div class="body">
1075  
1076        
1077        
1078        
1079          <p>Yesterday was my last day at Twitter. I left to start my own company. What I'll be working on is very exciting (though I'm keeping it secret for now).</p>
1080
1081<p>Leaving Twitter was a tough decision. I worked with a whole bunch of great people on fascinating problems with some of the most interesting data in the world. Ultimately though, I felt that if I didn't make this move, I would <a href="http://www.youtube.com/watch?v=jwG_qR6XmDQ">regret it</a> for the rest of my life. So I put in my papers about a month ago and then spent a month transitioning my team for my departure.</p>
1082
1083<p>This ends an eventful three years that started with me joining BackType in January of 2010. So much has happened in these past three years. I open-sourced <a href="https://github.com/nathanmarz/cascalog/wiki">Cascalog</a>, <a href="https://github.com/nathanmarz/elephantdb">ElephantDB</a>
1083, and <a href="http://storm-project.net">Storm</a>, started writing <a href="http://manning.com/marz">a book</a>, gave <a href="http://nathanmarz.com/speaking/">a lot of talks</a>, and in July of 2011 experienced the thrill of <a href="http://techcrunch.com/2011/07/05/twitter-acquires-social-analytics-startup-backtype/">being acquired</a>. My projects spread beyond BackType and Twitter to be relied on by <a href="https://github.com/nathanmarz/storm/wiki/Powered-By">dozens and dozens of companies</a>. Through all this, I learned an enormous amount about entrepreneurship, product development, marketing, recruiting, and project management.</p>
1084
1085<p>Stay tuned.</p>
1086
1087
1088        
1089  
1090          
1091  
1092         
1093  
1094      </div>
1095
1096    
1097
1098  </div>
1099
1100      <div class="journal-entry-tag journal-entry-tag-post-body">
1101      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/leaving-twitter.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />1 Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fleaving-twitter.html','Leaving%20Twitter');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1102      <div class="journal-entry-tag-post-body-line2"></div>
1103      <div class="journal-entry-tag-post-body-line3"></div>
1104    </div>
1105  
1106  
1107  <div class="clearer"></div>
1108
1109</div>
1110</div>
1111
1112
1113
1114
1115
1116
1117
1118<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1119<div id="item29131983" class="journal-entry">
1120
1121  <div class="journal-entry-float-day"><span class="name">Wednesday</span></div><div class="journal-entry-float-date"><span class="month">Sep</span><span class="date">19</span><span class="year">2012</span></div>
1122  
1123  
1124  
1125  <div class="journal-entry-text">
1126  
1127      
1128      <h2 class="title">
1129    
1130        
1131    
1132        <a class="journal-entry-navigation-current" href="/blog/storms-1st-birthday.html">Storm's 1st&nbsp;birthday</a>  
1133     
1134         
1135    
1136      </h2>
1137      
1138       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Wednesday, September 19, 2012</span> </div> 
1139
1140    
1141  
1142      
1143      <div class="body">
1144  
1145        
1146        
1147        
1148          <p><a href="http://storm-project.net/">Storm</a> was open-sourced exactly one year ago today. It's been an action-packed year for Storm, to say the least. Here's some of the exciting stuff that's happened over the past year:</p>
1149<ul>
1150<li>27 companies have publicized that they're <a href="https://github.com/nathanmarz/storm/wiki/Powered-By">using Storm in production</a>. I know of at least a few more companies using it that haven't published anything yet.</li>
1151<li>O'Reilly <a href="http://shop.oreilly.com/product/0636920024835.do">published a book</a> on Storm.</li>
1152<li>The&nbsp;<a href="https://groups.google.com/group/storm-user">Storm mailing list</a>&nbsp;has over 1300 members, with over 500 messages per month.</li>
1153<li>The&nbsp;<a href="https://twitter.com/stormprocessor">@stormprocessor</a>&nbsp;account has over 1200 followers.</li>
1154<li>More than 4000 people have starred the project <a href="https://github.com/nathanmarz/storm">on Github</a>.</li>
1155<li>There's a&nbsp;<a href="http://www.meetup.com/Bay-Area-Storm-Users/">regular Storm meetup</a>&nbsp;in the Bay Area with over 230 members. I've also seen lots of Storm-focused meetups happen all over the world over the past year.</li>
1156<li>29 people all over the world have contributed to the codebase</li>
1157<li>We released <a href="http://engineering.twitter.com/2012/08/trident-high-level-abstraction-for.html">Trident</a>, a high level abstraction for realtime computation, that is a major leap forward in what's possible in realtime.</li>
1158<li>Libraries have been released integrating Storm with Kestrel, Kafka, JMS, Cassandra, Memcached, and many more systems. For many, Storm is becoming the system of choice for connecting these systems together.</li>
1159<li>Storm's performance has been increased by over 10x. I've benchmarked it at 1M messages per second per node on an internal Twitter cluster.</li>
1160</ul>
1161<p>What I overwhelmingly hear from people is that they like Storm because it's simple to understand, flexible, and extremely robust in production. These have always been some of the core design goals of Storm, so I'm glad that we were able to succeed on these points.</p>
1162<p>We've got lots of exciting stuff planned over the next year. We have <a href="https://github.com/nathanmarz/storm/issues/305">a new metrics system</a> in development which will let you get deep insight into what's happening throughout your topology in realtime. And we have big plans for improving Trident and integrating it with more datastores and input sources.</p>
1163<p>Happy birthday Storm!</p>
1164
1165        
1166  
1167          
1168  
1169         
1170  
1171      </div>
1172
1173    
1174
1175  </div>
1176
1177      <div class="journal-entry-tag journal-entry-tag-post-body">
1178      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/storms-1st-birthday.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fstorms-1st-birthday.html','Storm%27s%201st%20birthday');">
1178 <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1179      <div class="journal-entry-tag-post-body-line2"></div>
1180      <div class="journal-entry-tag-post-body-line3"></div>
1181    </div>
1182  
1183  
1184  <div class="clearer"></div>
1185
1186</div>
1187</div>
1188
1189
1190
1191
1192
1193
1194
1195<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1196<div id="item14876847" class="journal-entry">
1197
1198  <div class="journal-entry-float-day"><span class="name">Monday</span></div><div class="journal-entry-float-date"><span class="month">Feb</span><span class="date">06</span><span class="year">2012</span></div>
1199  
1200  
1201  
1202  <div class="journal-entry-text">
1203  
1204      
1205      <h2 class="title">
1206    
1207        
1208    
1209        <a class="journal-entry-navigation-current" href="/blog/suffering-oriented-programming.html">Suffering-oriented&nbsp;programming</a>  
1210     
1211         
1212    
1213      </h2>
1214      
1215       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Monday, February 6, 2012</span> </div> 
1216
1217    
1218  
1219      
1220      <div class="body">
1221  
1222        
1223        
1224        
1225          <p>Someone asked me an interesting question the other day: "How did you justify taking such a huge risk on building <a href="https://github.com/nathanmarz/storm">Storm</a> while working on a <a href="http://backtype.com">startup</a>?" (Storm is a realtime computation system). I can see how from an outsider's perspective investing in such a massive project seems extremely risky for a startup. From my perspective, though, building Storm wasn't risky at all. It was challenging, but not risky.</p>
1226
1227<p>I follow a style of development that greatly reduces the risk of big projects like Storm. I call this style "suffering-oriented programming." Suffering-oriented programming can be summarized like so: don't build technology unless you feel the pain of not having it. It applies to the big, architectural decisions as well as the smaller everyday programming decisions. Suffering-oriented programming greatly reduces risk by ensuring that you're always working on something important, and it ensures that you are well-versed in a problem space before attempting a large investment.</p>
1228
1229<p>I have a mantra for suffering-oriented programming: "First make it possible. Then make it beautiful. Then make it fast."</p>
1230
1231<h3>First make it possible</h3>
1232
1233<p>When encountering a problem domain with which you're unfamiliar, it's a mistake to try to build a "general" or "extensible" solution right off the bat. You just don't understand the problem domain well enough to anticipate what your needs will be in the future. You'll make things generic that needn't be, adding complexity and wasting time.</p>
1234
1235<p>It's better to just "hack things out" and be very direct about solving the problems you have at hand. This allows you to get done what you need to get done and avoid wasted work. As you're hacking things out, you'll learn more and more about the intricacies of the problem space.</p>
1236
1237<p>The "make it possible" phase for Storm was one year of hacking out a stream processing system using queues and workers. We learned about guaranteeing data processing using an "ack" protocol. We learned to scale our realtime computations with clusters of queues and workers. We learned that sometimes you need to partition a message stream in different ways, sometimes randomly and sometimes using a hash/mod technique that makes sure the same entity always goes to the same worker. </p>
1238
1239<p>We didn't even know we were in the "make it possible" phase. We were just focused on building our products. The pain of the queues and workers system became acute very quickly though. Scaling the queues and workers system was tedious, and the fault-tolerance was nowhere near what we wanted. It was evident that the queues and workers paradigm was not at the right level of abstraction, as most of our code had to do with routing messages and serialization and not the actual business logic we cared about.</p>
1240
1241<p>At the same time, developing our product drove us to discover new use cases in the "realtime computation" problem space. We built a feature for our product that would compute the reach of a URL on Twitter. Reach is the number of unique people exposed to a URL on Twitter. It's a difficult computation that can require hundreds of database calls and tens of millions of impressions to distinct just for one computation. Our original implementation that ran on a single machine would take over a minute for hard URLs, and it was clear that we needed a distributed system of some sort to parallelize the computation to make it fast.</p>
1242
1243<p>One of the key realizations that sparked Storm was that the "reach problem" and the "stream processing" problem could be unified by a simple abstraction.</p>
1244
1245<h3>Then make it beautiful</h3>
1246
1247<p>You develop a "map" of the problem space as you explore it by hacking things out. Over time, you acquire more and more use cases within the problem domain and develop a deep understanding of the intricacies of building these systems. This deep understanding can guide the creation of "beautiful" technology to replace your existing systems, alleviate your suffering, and enable new systems/features that were too hard to build before.</p>
1248
1249<p>The key to developing the "beautiful" solution is figuring out the <strong>simplest</strong> set of abstractions that solve the concrete use cases you already have. It's a mistake to try to anticipate use cases you don't actually have or else you'll end up overengineering your solution. As a rule of thumb, the bigger the investment you're trying to make, the deeper you need to understand the problem domain and the more diverse your use cases need to be. Otherwise you risk the <a href="http://en.wikipedia.org/wiki/Second-system_effect">second-system effect</a>.</p>
1250
1251<p>"Making it beautiful" is where you use your design and abstraction skills to distill the problem space into simple abstractions that can be composed together. I view the development of beautiful abstractions as similar to statistical regression: you have a set of points on a graph (your use cases) and you're looking for the simplest curve that fits those points (a set of abstractions).</p>
1252
1253<div align=center><span class="full-image-block ssNonEditable"><span><img src="/storage/regression.png?__SQUARESPACE_CACHEVERSION=1328432737924" alt=""/></span></span></div>
1254
1255<p>The more use cases you have, the better you'll be able to find the right curve to fit those points. If you don't have enough points, you're likely to either overfit or underfit the graph, leading to wasted work and overengineering. </p>
1256
1257<p>A big part of making it beautiful is understanding the performance and resource characteristics of the problem space. This is one of the intricacies you learn in the "making it possible" phase, and you should take advantage of that learning when designing your beautiful solution.</p>
1258
1259<p>With Storm, I distilled the realtime computation problem domain into a small set of abstractions: streams, spouts, bolts, and topologies. I devised a new algorithm for guaranteeing data processing that eliminated the need for intermediate message brokers, the part of our system that caused the most complexity and suffering. That both stream processing and reach, two very different problems on the surface, mapped so elegantly to Storm was a strong indicator that I was onto something big. </p>
1260
1261<p>I took additional steps to acquire more use cases for Storm and validate my designs. I canvassed other engineers to learn about the particulars of the realtime problems they were dealing with. I didn't just ask people I knew. I also tweeted out that I was working on a new realtime system and wanted to learn about other people's use cases. This led to a lot of interesting discussions that educated me more on the problem domain and validated my design ideas.</p>
1262
1263<h3>Then make it fast</h3>
1264
1265<p>Once you've built out your beautiful design, you can safely invest time in profiling and optimization. Doing optimization too early will just waste time, because you still might rethink the design. This is called premature optimization.</p>
1266
1267<p>"Making it fast" isn't about the high level performance characteristics of a system. The understanding of those issues should have been acquired in the "make it possible" phase and designed for in the "make it beautiful" phase. "Making it fast" is about micro-optimizations and tightening up the code to be more resource efficient. So you might worry about things like asymptotic complexity in the "make it beautiful" phase and focus on the constant-time factors in the "make it fast" phase.</p>
1268
1269<h3>Rinse and repeat</h3>
1270
1271<p>Suffering-oriented programming is a continuous process. The beautiful systems you build give you new capabilities, which allow you to "make it possible" in new and deeper areas of the problem space. This feeds learning back to the technology. You often have to tweak or add to the abstractions you've already come up with to handle more and more use cases.</p>
1272
1273<p>Storm has gone through many iterations like this. When we first started using Storm, we discovered that we needed the capability to emit multiple, independent streams from a single component. We discovered that the addition of a special kind of stream called the "direct stream" would allow Storm to process batches of tuples as a concrete unit. Recently I developed "transactional topologies" which go beyond Storm's at-least-once processing guarantee and allow exactly-once messaging semantics to be achieved for nearly arbitrary realtime computation.</p>
1274
1275<p>By its nature, hacking things out in a problem domain you don't understand so well and constantly iterating can lead to some sloppy code. The most important characteristic of a suffering-oriented programmer is a relentless focus on refactoring. This is critical to prevent <a href="http://en.wikipedia.org/wiki/Accidental_complexity">accidental complexity</a> from sabotaging the codebase.</p>
1276
1277<h3>Conclusion</h3>
1278
1279<p>Use cases are everything in suffering-oriented programming. They're worth their weight in gold. The only way to acquire use cases is through gaining experience through hacking.</p>
1280
1281<p>There's a certain evolution most programmers go through. You start off struggling to get things to work and have absolutely no structure to your code. Code is sloppy and copy/pasting is prevalent. Eventually you learn about the benefits of structured programming and sharing logic as much as possible. Then you learn about making generic abstractions and using encapsulation to make it easier to reason about systems. Then you become obsessed with making all your co
1281de generic, with making things extensible to future-proof your programs.</p>
1282
1283<p>Suffering-oriented programming rejects that you can effectively anticipate needs you don't currently have. It recognizes that attempts to make things generic without a deep understanding of the problem domain will lead to complexity and waste. Designs must always be driven by real, tangible use cases.</p>
1284
1285<p><em>You should follow me on Twitter <a href="http://twitter.com/nathanmarz">here</a>.</em></p>
1286
1287
1288        
1289  
1290          
1291  
1292         
1293  
1294      </div>
1295
1296    
1297
1298  </div>
1299
1300      <div class="journal-entry-tag journal-entry-tag-post-body">
1301      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/suffering-oriented-programming.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fsuffering-oriented-programming.html','Suffering-oriented%20programming');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1302      <div class="journal-entry-tag-post-body-line2"></div>
1303      <div class="journal-entry-tag-post-body-line3"></div>
1304    </div>
1305  
1306  
1307  <div class="clearer"></div>
1308
1309</div>
1310</div>
1311
1312
1313
1314
1315
1316
1317
1318<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1319<div id="item14510573" class="journal-entry">
1320
1321  <div class="journal-entry-float-day"><span class="name">Monday</span></div><div class="journal-entry-float-date"><span class="month">Jan</span><span class="date">09</span><span class="year">2012</span></div>
1322  
1323  
1324  
1325  <div class="journal-entry-text">
1326  
1327      
1328      <h2 class="title">
1329    
1330        
1331    
1332        <a class="journal-entry-navigation-current" href="/blog/early-access-edition-of-my-book-is-available.html">Early access edition of my book is&nbsp;available</a>  
1333     
1334         
1335    
1336      </h2>
1337      
1338       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Monday, January 9, 2012</span> </div> 
1339
1340    
1341  
1342      
1343      <div class="body">
1344  
1345        
1346        
1347        
1348          <p>The <a href="http://www.manning.com/marz/">early access edition</a> of my book <em>Big Data: principles and best practices of scalable realtime data systems</em> is now available from Manning! I've been working on this book for quite some time, and I'm excited to have it out there and start getting some feedback.</p>
1349<p>The interest in the book has already been overwhelming, and I've been answering questions about it <a href="http://news.ycombinator.com/item?id=3444300">on Hacker News</a>.</p>
1350
1351        
1352  
1353          
1354  
1355         
1356  
1357      </div>
1358
1359    
1360
1361  </div>
1362
1363      <div class="journal-entry-tag journal-entry-tag-post-body">
1364      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/early-access-edition-of-my-book-is-available.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />1 Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fearly-access-edition-of-my-book-is-available.html','Early%20access%20edition%20of%20my%20book%20is%20available');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1365      <div class="journal-entry-tag-post-body-line2"></div>
1366      <div class="journal-entry-tag-post-body-line3"></div>
1367    </div>
1368  
1369  
1370  <div class="clearer"></div>
1371
1372</div>
1373</div>
1374
1375
1376
1377
1378
1379
1380
1381<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1382<div id="item13111114" class="journal-entry">
1383
1384  <div class="journal-entry-float-day"><span class="name">Thursday</span></div><div class="journal-entry-float-date"><span class="month">
1384Oct</span><span class="date">13</span><span class="year">2011</span></div>
1385  
1386  
1387  
1388  <div class="journal-entry-text">
1389  
1390      
1391      <h2 class="title">
1392    
1393        
1394    
1395        <a class="journal-entry-navigation-current" href="/blog/how-to-beat-the-cap-theorem.html">How to beat the CAP&nbsp;theorem</a>  
1396     
1397         
1398    
1399      </h2>
1400      
1401       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Thursday, October 13, 2011</span> </div> 
1402
1403    
1404  
1405     
1406
1407      <div class="body">
1408  
1409        
1410  
1411         
1412  
1413        <p><p>The CAP theorem states a database cannot guarantee consistency, availability, and partition-tolerance at the same time. But you can't sacrifice partition-tolerance (see <a href="http://codahale.com/you-cant-sacrifice-partition-tolerance/">here</a> and <a href="http://www.cloudera.com/blog/2010/04/cap-confusion-problems-with-partition-tolerance/">here</a>), so you must make a tradeoff between availability and consistency. Managing this tradeoff is a central focus of the NoSQL movement.</p>
1414
1415<p>Consistency means that after you do a successful write, future reads will always take that write into account. Availability means that you can always read and write to the system. During a partition, you can only have one of these properties.</p>
1416
1417<p>Systems that choose consistency over availability have to deal with some awkward issues. What do you do when the database isn't available? You can try buffering writes for later, but you risk losing those writes if you lose the machine with the buffer. Also, buffering writes can be a form of inconsistency because a client thinks a write has succeeded but the write isn't in the database yet. Alternatively, you can return errors back to the client when the database is unavailable. But if you've ever used a product that told you to "try again later", you know how aggravating this can be. </p>
1418
1419<p>The other option is choosing availability over consistency. The best consistency guarantee these systems can provide is "eventual consistency". If you use an eventually consistent database, then sometimes you'll read a different result than you just wrote. Sometimes multiple readers reading the same key at the same time will get different results. Updates may not propagate to all replicas of a value, so you end up with some replicas getting some updates and other replicas getting different updates. It is up to you to repair the value once you detect that the values have diverged. This requires tracing back the history using vector clocks and merging the updates together (called "read repair").</p>
1420
1421<p>I believe that maintaining eventual consistency in the application layer is too heavy of a burden for developers. Read-repair code is extremely susceptible to developer error; if and when you make a mistake, faulty read-repairs will introduce irreversible corruption into the database.</p>
1422
1423<p>So sacrificing availability is problematic and eventual consistency is too complex to reasonably build applications. Yet these are the only two options, so it seems like I'm saying that you're damned if you do and damned if you don't. The CAP theorem is a fact of nature, so what alternative can there possibly be?</p>
1424
1425<p>There is another way. You can't avoid the CAP theorem, but you can isolate its complexity and prevent it from sabotaging your ability to reason about your systems. The complexity caused by the CAP theorem is a symptom of fundamental problems in how we approach building data systems. Two problems stand out in particular: the use of mutable state in databases and the use of incremental algorithms to update that state. It is the interaction between these problems and the CAP theorem that causes complexity.</p>
1426
1427<p>In this post I'll show the design of a system that beats the CAP theorem by preventing the complexity it normally causes. But I won't stop there. The CAP theorem is a result about the degree to which data systems can be fault-tolerant to machine failure. Yet there's a form of fault-tolerance that's much more important than machine fault-tolerance: human fault-tolerance. If there's any certainty in software development, it's that developers aren't perfect and bugs will inevitably reach production. Our data systems must be resilient to buggy programs that write bad data, and the system I'm going to show is as human fault-tolerant as you can get.</p>
1428
1429<p>This post is going to challenge your basic assumptions on how data systems should be built. But by breaking down our current ways of thinking and re-imagining how data systems should be built, what emerges is an architecture more elegant, scalable, and robust than you ever thought possible.</p>
1430</p>
1431  
1432        <p class="journal-read-more-tag"><a href="/blog/how-to-beat-the-cap-theorem.html">
1432Click to read more ...</a></p>
1433  
1434      </div>
1435  
1436    
1437
1438  </div>
1439
1440      <div class="journal-entry-tag journal-entry-tag-post-body">
1441      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/how-to-beat-the-cap-theorem.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />4 Comments</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fhow-to-beat-the-cap-theorem.html','How%20to%20beat%20the%20CAP%20theorem');"> <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1442      <div class="journal-entry-tag-post-body-line2"></div>
1443      <div class="journal-entry-tag-post-body-line3"></div>
1444    </div>
1445  
1446  
1447  <div class="clearer"></div>
1448
1449</div>
1450</div>
1451
1452
1453
1454
1455
1456
1457
1458<div class="journal-entry-wrapper post-text authored-by-nmarz ">
1459<div id="item10991590" class="journal-entry">
1460
1461  <div class="journal-entry-float-day"><span class="name">Tuesday</span></div><div class="journal-entry-float-date"><span class="month">Mar</span><span class="date">29</span><span class="year">2011</span></div>
1462  
1463  
1464  
1465  <div class="journal-entry-text">
1466  
1467      
1468      <h2 class="title">
1469    
1470        
1471    
1472        <a class="journal-entry-navigation-current" href="/blog/my-talks-at-posscon.html">My talks at&nbsp;POSSCON</a>  
1473     
1474         
1475    
1476      </h2>
1477      
1478       <div class="journal-entry-tag journal-entry-tag-post-title"><span class="posted-on">   <img title="Date" alt="Date" class="inline-icon date-icon" rel="dark" src="/universal/images/transparent.png" />Tuesday, March 29, 2011</span> </div> 
1479
1480    
1481  
1482      
1483      <div class="body">
1484  
1485        
1486        
1487        
1488          <p>Last week I went to POSSCON in Columbia, South Carolina. It was an interesting experience and a good reminder that not everyone in the world thinks like we do in Silicon Valley.</p>
1489<p>I gave two talks at the conference. One was a technical talk about how to build realtime Big Data systems, and the other was a non-technical talk about the things we do at BackType to be a super-productive team. Both slide decks are embedded below.</p>
1490<div id="__ss_7389389" style="width: 425px;"><strong><a title="The Secrets of Building Realtime Big Data Systems" href="http://www.slideshare.net/nathanmarz/the-secrets-of-building-realtime-big-data-systems">The Secrets of Building Realtime Big Data Systems</a></strong> <object id="__sse7389389" width="425" height="355"> <param name="movie" value="http://static.slidesharecdn.com/swf/ssplayer2.swf?doc=posscon-big-data-systems-110325124539-phpapp01&stripped_title=the-secrets-of-building-realtime-big-data-systems&userName=nathanmarz" /> <param name="allowFullScreen" value="true"/> <param name="allowScriptAccess" value="always"/> <embed name="__sse7389389" src="http://static.slidesharecdn.com/swf/ssplayer2.swf?doc=posscon-big-data-systems-110325124539-phpapp01&stripped_title=the-secrets-of-building-realtime-big-data-systems&userName=nathanmarz" type="application/x-shockwave-flash" allowscriptaccess="always" allowfullscreen="true" width="425" height="355"></embed> </object>
1491<div id="__ss_7419561" style="width: 425px;"><strong><a title="Become Efficient or Die: The Story of BackType" href="http://www.slideshare.net/nathanmarz/become-efficient-or-die-the-story-of-backtype">Become Efficient or Die: The Story of BackType</a></strong> <object id="__sse7419561" width="425" height="355"> <param name="movie" value="http://static.slidesharecdn.com/swf/ssplayer2.swf?doc=backtype-efficiency-110328132458-phpapp02&stripped_title=become-efficient-or-die-the-story-of-backtype&userName=nathanmarz" /> <param name="allowFullScreen" value="true"/> <param name="allowScriptAccess" value="always"/> <embed name="__sse7419561" src="http://static.slidesharecdn.com/swf/ssplayer2.swf?doc=backtype-efficiency-110328132458-phpapp02&stripped_title=become-efficient-or-die-the-story-of-backtype&userName=nathanmarz" type="application/x-shockwave-flash" allowscriptaccess="always" allowfullscreen="true" width="425" height="355"></embed> </object></div>
1492</div>
1493
1494        
1495  
1496          
1497  
1498         
1499  
1500      </div>
1501
1502    
1503
1504  </div>
1505
1506      <div class="journal-entry-tag journal-entry-tag-post-body">
1507      <div class="journal-entry-tag-post-body-line1"><span class="post-comments"><a href="/blog/my-talks-at-posscon.html#comments"> <img title="Comment" alt="Comment" class="inline-icon comment-icon" rel="dark" src="/universal/images/transparent.png" />Post a Comment</a></span> |  <span class="share-item"><a href="javascript:noop();" onclick="Squarespace.Interaction.shareLink(this,'%2Fblog%2Fmy-talks-at-posscon.html','My%20talks%20at%20POSSCON');">
1507 <img title="Share Article" alt="Share Article" class="inline-icon share-icon" rel="dark" src="/universal/images/transparent.png" />Share Article </a></span> </div>
1508      <div class="journal-entry-tag-post-body-line2"></div>
1509      <div class="journal-entry-tag-post-body-line3"></div>
1510    </div>
1511  
1512  
1513  <div class="clearer"></div>
1514
1515</div>
1516</div>
1517
1518
1519
1520
1521
1522    
1523    
1524  </div>
1525
1526  
1527  <div  class="paginationControlWrapper journal-navigation" >
1528
1529    <!-- 11 - 20 of 50 (page 2 of 5) -->
1530
1531    <span class="paginationControlPrefix">
1532      
1533              Page
1534       
1535    </span>
1536    
1537    
1538
1539    
1540              <a class="paginationPageNumber"  href="/blog/" >1</a>
1541      
1542    
1543              <a class="paginationPageNumber activePage" href="javascript:noop();">2</a>
1544      
1545    
1546              <a class="paginationPageNumber"  href="/blog/?currentPage=3" >3</a>
1547      
1548    
1549              <a class="paginationPageNumber"  href="/blog/?currentPage=4" >4</a>
1550      
1551    
1552              <a class="paginationPageNumber"  href="/blog/?currentPage=5" >5</a>
1553      
1554     
1555
1556          <span class="paginationEllipsis">...</span> <a class="paginationPageNumber"  href="/blog/?currentPage=5" >5</a>
1557    
1558
1559          <span class="paginationControlNextPageSuffix">
1560        <a  href="/blog/?currentPage=3" >
1561          Next 10 Entries »
1562        </a>
1563      </span>
1564    
1565    
1566    
1567
1568  </div>
1569
1570
1571
1572  
1573
1574
1575</div></div> <!-- Content -->
1576  
1577    
1578  
1579    </div></div> <!-- Page Body -->
1580
1581    
1582    
1583    <div id="pageFooterWrapper"><div id="pageFooter">
1584  
1585              Copyright &copy; 2012-2019, Nathan Marz.  All rights reserved.
1586
1587<script type="text/javascript" charset="utf-8">
1588                        var disqus_forum_url = 'thoughtsfromtheredplanet';
1589                    </script>
1589
1590                    
1590<script type="text/javascript" src="http://mediacdn.disqus.com/1362167690/embed/squarespace.js"></script>
1590
1591                    
1591<script type="text/javascript" src="http://disqus.com/forums/thoughtsfromtheredplanet/embed.js"></script>
1591
1592                
1593
1594
1595<script type="text/javascript">
1596
1597  var _gaq = _gaq || [];
1598  _gaq.push(['_setAccount', 'UA-12231066-1']);
1599  _gaq.push(['_trackPageview']);
1600
1601  (function() {
1602    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
1603    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
1604    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
1605  })();
1606
1607</script>
1607
1608
1609<script type="text/javascript">    
1610         SyntaxHighlighter.defaults['toolbar'] = false;      
1611          SyntaxHighlighter.all();
1612        </script>
1612 
1613      
1614  
1615      
1616  
1617    </div></div>
1618  
1619  </div></div> <!-- Canvas -->
1620  
1621   
1622
1623
1624  <div class="clearer" id="bodyClearer"></div>
1625
1626  
1627
1628
1629
1630</body>
1631</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.