PageSourceSearch

https://thoppe.github.io/Presentation-Black-Hack-Data-Wrangling/BHDW_DW_meetup.html

html thoppe.github.io collected 2026-10-03 10:04:46 UTC 31,989 bytes, 753 lines download raw bytes

1<!doctype html>
2<html lang="en">
3
4  <head>
5    <meta charset="utf-8">
6
7    <title>Black Hat Data Wrangling</title>
8    
9    <meta name="description" content="">
10    <meta name="author" content="Travis Hoppe & Robert Dempsey">
11
12    <meta name="apple-mobile-web-app-capable" content="yes" />
13    <meta name="apple-mobile-web-app-status-bar-style" 
14	  content="black-translucent" />
15
16    <meta name="viewport" content="width=device-width, 
17				   initial-scale=1.0, 
18				   maximum-scale=1.0, 
19				   user-scalable=no">
20
21    <link rel="stylesheet" href="reveal.js/css/reveal.css">
22
23    <!-- Default themes -->
24    <link rel="stylesheet" href="reveal.js/css/theme/simple.css" id="theme">
25    <link rel="stylesheet" href="reveal.js/css/theme/night.css" id="theme">
26
27    <!-- For syntax highlighting -->
28    <link rel="stylesheet" href="reveal.js/lib/css/zenburn.css">
29
30    <!-- For LaTeX formating highlighting -->
31    <link rel="stylesheet" href="md2reveal/css/md2reveal_style.css">
32
33    <!--[if lt IE 9]>
34	
34<script src="reveal.js/lib/js/html5shiv.js"></script>
34
35	<![endif]-->
36  </head>
37
38  <body>
39
40    <div class="reveal"><div class="slides">
41	<section class="vertical-stack">
42<section   class="vertical-slide">
43<p>
44</p>
45<br>
46<p>
47<h1>Black Hat</h1><h1>Data Wrangling</h1></p>
48<hr>
49<p>
50<h3><a href="http://thoppe.github.io/">Travis Hoppe</a> / <a href="http://robertwdempsey.com/about-robert-dempsey/">Robert Dempsey</a></h3><a href="https://twitter.com/metasemantic">@metasemantic</a> / <a href="https://twitter.com/rdempsey">@rdempsey</a></p>
51<br>
52<p>
53<div class="reference_footnote"><a href="http://thoppe.github.io/Presentation-Black-Hack-Data-Wrangling">http://thoppe.github.io/Presentation-Black-Hack-Data-Wrangling</a></div><br><style>.whitehat {background:#333333;padding-bottom:0px !important;}</style></p>
54</section>
55</section><section class="vertical-stack">
56<section   class="vertical-slide">
57<p>
58<h1>Data Wranglers</h1><h2>Want your data!</h2></p>
59<br>
60<figure>
61    <a href="images/data_ST.gif">
62    <img class="" src="images/data_ST.gif" height="300px">
63    </a>
64    
65    </figure>
66<figure>
67    <a href="images/fast_type.gif">
68    <img class="" src="images/fast_type.gif" height="300px">
69    </a>
70    
71    </figure>
72<br>
73<p>
74<h2>How can you <strong><strike>stop</strike></strong> slow them?</h2></p>
75</section><section   class="vertical-slide">
76<p>
77<h2>White-hat data wrangler</h2>Working hard to make your data <strong>accessible</strong> to others.</p>
78<figure>
79    <a href="images/dcdw_logo.jpg">
80    <img class="transparent_image" src="images/dcdw_logo.jpg" height="200px">
81    </a>
82    
83    </figure>
84<figure>
85    <a href="images/DSDC_logo.png">
86    <img class="transparent_image" src="images/DSDC_logo.png" height="200px">
87    </a>
88    
89    </figure>
90<figure>
91    <a href="images/nyc_dw_logo.png">
92    <img class="transparent_image" src="images/nyc_dw_logo.png" height="200px">
93    </a>
94    
95    </figure>
96<br>
97<br>
98<br>
99<br>
100<p>
101<em>But what if you don't want people to have your data...?</em></p>
102</section><section   class="vertical-slide">
103<p>
104<h2>Black-hat data wrangler</h2>Working hard to make your data as <strong>inaccessible</strong> as possible.</p>
105<br>
106<figure>
107    <a href="images/evil_mrburns.png">
108    <img class="transparent_image" src="images/evil_mrburns.png" height="200px">
109    </a>
110    
111    </figure>
112<figure>
113    <a href="images/evil_cards.jpg">
114    <img class="transparent_image" src="images/evil_cards.jpg" height="200px">
115    </a>
116    
117    </figure>
118<figure>
119    <a href="images/evil_umbrella2.png">
120    <img class="transparent_image" src="images/evil_umbrella2.png" height="200px">
121    </a>
122    
123    </figure>
124<br>
125<br>
126<p>
127What kind of data?</p>
128<br>
129<p>
130<em>Corporate finance / Political Donations / Regulations /</em><br><em> Anti-FOIA / Digital Marketer / ...</em></p>
131</section><section   class="vertical-slide">
132<p>
133<h2>Why not disconnect from the net?</h2></p>
134<br>
135<br>
136<figure>
137    <a href="images/drevil2.jpg">
138    <img class="" src="images/drevil2.jpg" height="300px">
139    </a>
140    
141    </figure>
142<figure>
143    <a href="images/drevil.jpg">
144    <img class="" src="images/drevil.jpg" height="300px">
145    </a>
146    
147    </figure>
148<figure>
149    <a href="images/drevil3.jpg">
150    <img class="" src="images/drevil3.jpg" height="300px">
151    </a>
152    
153    </figure>
154</section><section   class="vertical-slide">
155<p>
156<h1>When do you need a</h1><h2><em>Black Hack Data Wrangler?</em></h2></p>
157<br>
158<p>
159<h3>You have a <strong>large</strong> amount of data</h3><h3>The data must be made <strong>public</strong></h3><h3>Should be human readable but <strong>computer-unfriendly</strong></h3><h3>Your actions should be <strong>hidden</strong> to a casual user</h3></p>
160</section>
161</section><section class="vertical-stack">
162<section   class="vertical-slide">
163<p>
164<h3>Presentation format</h3><h5>Hack quantification</h5></p>
165<br>
166<p>
167implementation : <strong>EASY</strong> : <strong>MEDIUM</strong> : <strong>HARD</strong> : <strong>WTF</strong></p>
168<br>
169<p>
170defense : <strong>WEAK</strong> : <strong>REASONABLE</strong> : <strong>STRONG</strong> : <strong>RIDICULOUS</strong></p>
171<br>
172<p>
173hack-level : <strong>SCRIPT-KIDDIE</strong> : <strong>CORPORATE</strong> : <strong>HOLLYWOOD</strong></p>
174</section>
175</section><section class="vertical-stack">
176<section   class="vertical-slide">
177<p>
178<h3>Table of Contents</h3></p>
179<br>
180<p>
181<h4><a href="#Hack1">Disable right-click</a> [T]</h4><h4><a href="#Hack2">Minification</a> [R]</h4><h4><a href="#Hack3">Authentication</a> [T]</h4><h4><a href="#Hack4">Data limits</a> [R]</h4><h4><a href="#Hack5">Rendering to images</a> [R]</h4><h4><a href="#Hack6">JavaScript page links</a> [T]</h4><h4><a href="#Hack7">Watermarking</a> [R]</h4><h4><a href="#Hack8">Honeypots & Stenography</a> [T]</h4><h4><a href="#Hack9">Remove markup metadata</a> [T]</h4><h4><a href="#Hack10">HTML obfuscation</a> [R]</h4><h4><a href="#Hack11">Serving HTML as PDF</a> [R]</h4><h4><a href="#Hack12">Text remapping</a> [T]</h4></p>
182</section>
183</section><section class="vertical-stack">
184<section id="Hack1"  class="vertical-slide">
185<p>
186<h2><code>BlackHat1</code>: Disable right-click</h2>implementation <strong>EASY</strong> : defense <strong>WEAK</strong> : hack-level <strong>SCRIPT-KIDDIE</strong></p>
187<br>
188<pre><code class=python>&lt;script language=&quot;javascript&quot;&gt;
189document.onmousedown=disableclick;
190status=&quot;Right Click Disabled&quot;;
191function disableclick(event) {
192  if(event.button==2) {
193     alert(status);
194     return false;    
195} }
196&lt;/script&gt;</code></pre>
197<br>
198<p>
199Also in this category, CSS overlays.</p>
200</section><section   class="vertical-slide">
201<p>
202<div class="whitehat"><h2><code>WhiteHat1</code>: Disable right-click </div></h2></p>
203<br>
204<p>
205<h3>Open developers console (F12), search for <code>disableclick</code> and remove.</h3></p>
206<br>
207<p>
208<h3>Turn of javascript.</h3></p>
209<br>
210<p>
211<h3>Use a headless (or mobile) browser.</h3></p>
212</section>
213</section><section class="vertical-stack">
214<section id="Hack2"  class="vertical-slide">
215<p>
216<h2><code>BlackHat2</code>: Minification</h2>implementation <strong>EASY</strong> : defense <strong>WEAK</strong> : hack-level <strong>SCRIPT-KIDDIE</strong></p>
217<br>
218<p>
219<a href="https://github.com/kangax/html-minifier">Kangax HTML Minifier</a>: removes comments, whitespace, empty elements, and much more. Also minifies javascript and CSS. Ruby wrapper: <a href="https://github.com/stereobooster/html_minifier">html_minifier</a><h4>INPUT</h4></p>
220<pre><code class=python>&lt;div class=&quot;reveal&quot;&gt;
221    &lt;div class=&quot;slides&quot;&gt;
222        &lt;section class=&quot;vertical-stack&quot;&gt;
223            &lt;section class=&quot;vertical-slide&quot;&gt;
224                &lt;h1&gt;Black Hat&lt;/h1&gt;
225                &lt;h1&gt;Data Wrangling&lt;/h1&gt;
226                &lt;hr&gt;
227                &lt;h3&gt;&lt;a href=&quot;http://thoppe.github.io/&quot;&gt;Travis Hoppe&lt;/a&gt; /
228                &lt;a href=
229                &quot;http://robertwdempsey.com/about-robert-dempsey/&quot;&gt;Robert
230                Dempsey&lt;/a&gt;&lt;/h3&gt;&lt;a href=
231                &quot;https://twitter.com/metasemantic&quot;&gt;@metasemantic&lt;/a&gt; /
232                &lt;a href=&quot;https://twitter.com/rdempsey&quot;&gt;@rdempsey&lt;/a&gt;
233                &lt;br&gt;
234            &lt;/section&gt;
235        &lt;/section&gt;
236    &lt;/div&gt;
237&lt;/div&gt;</code></pre>
238<p>
239<h4>OUTPUT</h4></p>
240<pre><code class=python>&lt;div class=reveal&gt;&lt;
240div class=slides&gt;&lt;section class=vertical-stack&gt;&lt;section class=vertical-slide&gt;&lt;h1&gt;Black Hat&lt;/h1&gt;&lt;h1&gt;Data Wrangling&lt;/h1&gt;&lt;hr&gt;&lt;h3&gt;&lt;a href=&quot;http://thoppe.github.io/&quot;&gt;Travis Hoppe&lt;/a&gt; / &lt;a href=&quot;http://robertwdempsey.com/about-robert-dempsey/&quot;&gt;Robert Dempsey&lt;/a&gt;&lt;/h3&gt;&lt;a href=https://twitter.com/metasemantic&gt;@metasemantic&lt;/a&gt; / &lt;a href=https://twitter.com/rdempsey&gt;@rdempsey&lt;/a&gt;&lt;p&gt;&lt;/p&gt;&lt;br&gt;&lt;/section&gt;&lt;/section&gt;&lt;/div&gt;&lt;/div&gt;</code></pre>
241</section><section   class="vertical-slide">
242<p>
243<div class="whitehat"><h2><code>WhiteHat2</code>: Minification </div></h2>De-minify the HTML using freely available tools.</p>
244<br>
245<p>
246Online tools: <a href="http://unminify.com/">Unminify</a>, <a href="">JS Beautifier</a><br><em>or</em><br>Text editor: <a href="https://github.com/welovewordpress/SublimeHtmlTidy">HTML Tidy</a> (Sublime Text)<br><em>or</em><br>Automate it: <a href="https://github.com/beautify-web/js-beautify">JS Beautifier</a></p>
247<pre><code class=python>$ pip install jsbeautifier
248$ js-beautify file.js</code></pre>
249</section>
250</section><section class="vertical-stack">
251<section id="Hack3"  class="vertical-slide">
252<p>
253<h2><code>BlackHat3</code>: Authentication</h2>implementation <strong>MEDIUM</strong> : defense <strong>REASONABLE</strong> : hack-level <strong>CORPORATE</strong></p>
254<br>
255<p>
256<h4>not RESTful?</h4>Implement visitor control via <code>$SESSIONS</code>. Give every new visitor to the site a unique ID that you control and limit access with. Bonus, restrict user-agent.</p>
257<br>
258<p>
259<h4>REST API?</h4>Require all meaningful data requests to go through OAuth2, cumbersome for new-comers and direct control over the data distribution.</p>
260</section><section   class="vertical-slide">
261<p>
262<div class="whitehat"><h2><code>WhiteHat3</code>: Authentication </div></h2></p>
263<br>
264<p>
265Create session ID's with headless browsers<br><em>and</em><br>simulate user-agents</p>
266<br>
267<p>
268Black Hat Warning: Poorly designed session states<br>(that don't clear and hold large internal variables) can DoS your server!</p>
269</section>
270</section><section class="vertical-stack">
271<section id="Hack4"  class="vertical-slide">
272<p>
273<h2><code>BlackHat4:</code> Data & time limits</h2>implementation <strong>MEDIUM</strong> : defense <strong>REASONABLE</strong> : hack-level <strong>CORPORATE</strong></p>
274<br>
275<p>
276Detection: high download rates or unusual traffic within a given timespan;<br>all traffic from a single client or IP address.</p>
277<br>
278<p>
279Rate limit individual IP addresses or a specific id.<br>Delay content delivery.<br>Return HTTP 301, 40x or 50x errors (<a href="http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html">full list</a>)</p>
280</section><section   class="vertical-slide">
281<p>
282<div class="whitehat"><h2><code>WhiteHat4</code>: Data & time limits </div></h2></p>
283<br>
284<p>
285Cycle your IP address using <a href="http://robertwdempsey.com/hma">VPN/proxy services</a> or <a href="https://www.torproject.org/">TOR</a> (see <a href="http://thoppe.github.io/tor_spiders/HnC_presentation.html#/">TOR spiders</a>).<br><em>and</em><br>Slow down your scraper: Scrapy <a href="http://doc.scrapy.org/en/latest/topics/autothrottle.html">autothrottle</a>, custom timing code<br><em>and</em><br>Change your user agent: Scrapy <a href="https://github.com/cnu/scrapy-random-useragent">random user agent</a>, <a href="http://stackoverflow.com/questions/24226781/changing-user-agent-in-python-3-for-urrlib-urlopen">custom Python code</a></p>
286</section>
287</section><section class="vertical-stack">
288<section id="Hack5"  class="vertical-slide">
289<p>
290<h2><code>BlackHat5</code>: Rendering to images</h2>implementation <strong>MEDIUM</strong> : defense <strong>STRONG</strong> : hack-level <strong>CORPORATE</strong></p>
291<br>
292<p>
293<h3>Text to Image</h3><a href="https://www.daftlogic.com/projects-text-to-image.htm">PHP Text to Image</a> / <a href="http://www.imagemagick.org/script/index.php">ImageMagick</a><br><em>or</em><br>Draw text onto an HTML5 canvas using <a href="http://jsfiddle.net/JUWrV/">JavaScript</a> / use the HTML5 <a href="http://stackoverflow.com/questions/27552969/convert-text-to-canvas-image-preserving-formatting">canvasElement.toDataURL</a> element</p>
294</section><section   class="vertical-slide">
295<p>
296<div class="whitehat"><h2><code>WhiteHat5</code>: Rendering to images </div></h2></p>
297<br>
298<p>
299Server or desktop-based OCR software<br><em>or</em><br>Adobe Acrobat: Image -> PDF -> OCR (manual)<br><em>or</em><br>Python: <a href="https://github.com/tmbdev/ocropy">OCRopus</a><br><em>or</em><br><a href="https://github.com/tesseract-ocr/tesseract">Tesseract Open Source OCR Engine</a></p>
300</section>
301</section><section class="vertical-stack">
302<section id="Hack6"  class="vertical-slide">
303<p>
304<h2><code>BlackHat6</code>: JavaScript page links</h2>implementation <strong>MEDIUM</strong> : defense <strong>REASONABLE</strong> : hack-level <strong>CORPORATE</strong><br>Infinite pagination/scroll. Ex. <a href="https://dribbble.com/">Dribble</a></p>
305<figure>
306    <a href="images/infinite_scroll.gif">
307    <img class="" src="images/infinite_scroll.gif" height="300px">
308    </a>
309    
310    </figure>
311<br>
312<p>
313Forces the user to simulate AJAX (stops headless browsers).<br><em>Combine with user sessions and data limits!</em></p>
314<br>
315<p>
316<a href="http://videolectures.net/chi08_kieras_phc/">Psychology in Human-Computer Interaction</a> by David Kieras<br>shows this frustrates the user with lack of control.</p>
317<br>
318<p>
319<div class="reference_footnote">Image from <a href="https://visualhierarchy.co/blog/wp-content/uploads/2015/09/infinite_scroll.gif">visualhierarchy</a></div></p>
320</section><section  data-background="images/infinite-hate.gif" class="vertical-slide">
321<p>
322<br><br><br><br><br><br><br><br><br><br><br><br><br><br><br><font color="black"></p>
323<br>
324<p>
325<div class="reference_footnote">Image from <a href="https://dribbble.com/shots/2152555-I-Hate-Infinite-Scroll">Per Vestman@Dribble</a></div><br></font></p>
326</section><section   class="vertical-slide">
327<p>
328<div class="whitehat"><h2><code>WhiteHat6</code>: JavaScript page links </div></h2></p>
329<br>
330<p>
331Don't emulate a browser, <em>be</em> the browser! Selenium ex.</p>
332<br>
333<pre><code class=python>from selenium import webdriver
334from selenium.webdriver.common.by import By
335driver = webdriver.Firefox()
336driver.get('http://www.google.com')
337
338q = driver.find_element(By.NAME, 'q')
339q.send_keys('Black Hat Data Wrangling')
340q.submit()</code></pre>
341</section>
342</section><section class="vertical-stack">
343<section id="Hack7"  class="vertical-slide">
344<p>
345<h2><code>BlackHat7</code>: Watermarking</h2>implementation <strong>EASY</strong> : defense <strong>STRONG</strong> : hack-level <strong>SCRIPT-KIDDIE</strong></p>
346<br>
347<p>
348<h4>Easy mode, simple IP protection</h4></p>
349<figure>
350    <a href="images/watermark_examples/shutterstock1.jpg">
351    <img class="" src="images/watermark_examples/shutterstock1.jpg" height="300">
352    </a>
353    <figcaption>Easy to remove.</figcaption>
354    </figure>
355<figure>
356    <a href="images/watermark_examples/shutterstock2.jpg">
357    <img class="" src="images/watermark_examples/shutterstock2.jpg" height="300">
358    </a>
359    <figcaption>Requires time to remove, not automated.</figcaption>
360    </figure>
361<br>
362<p>
363Can watermark non images too!</p>
364</section><section   class="vertical-slide">
365<p>
366<div class="whitehat"><h2><code>WhiteHat7</code>: Watermarking</h2></div></p>
367<br>
368<p>
369<h3>Simple removal</h3>Crop the picture in any photo editor<br><em>or</em><br>Use the restoration function in <a href="http://www.theinpaint.com/">Inpaint</a>: $20</p>
370<br>
371<p>
372<h3>More complex removal</h3>"Content Aware Fill" in Photoshop</p>
373</section><section   class="vertical-slide">
374<p>
375<h3>Cropping</h3></p>
376<figure>
377    <a href="images/watermark_examples/remove-text-photoshop-label.jpg">
378    <img class="" src="images/watermark_examples/remove-text-photoshop-label.jpg" height="300">
379    </a>
380    
381    </figure>
382<figure>
383    <a href="images/watermark_examples/remove-text-photoshop-label-crop.jpg">
384    <img class="" src="images/watermark_examples/remove-text-photoshop-label-crop.jpg" height="300">
385    </a>
386    
387    </figure>
388<br>
389<p>
390<h3>Content Aware Fill in Photoshop</h3></p>
391<figure>
392    <a href="images/watermark_examples/remove-watermark-photoshop-cross-overlay.jpg">
393    <img class="" src="images/watermark_examples/remove-watermark-photoshop-cross-overlay.jpg" height="300">
394    </a>
395    
396    </figure>
397<figure>
398    <a href="images/watermark_examples/remove-watermark-photoshop-cross-overlay-filled.jpg">
399    <img class="" src="images/watermark_examples/remove-watermark-photoshop-cross-overlay-filled.jpg" height="300">
400    </a>
401    
402    </figure>
403</section>
404</section><section class="vertical-stack">
405<section id="Hack8"  class="vertical-slide">
406<p>
407<h2><code>BlackHat8</code>: Honeypots & Steganography</h2>implementation <strong>HARD</strong> : defense <strong>RIDICULOUS</strong> : hack-level <strong>HOLLYWOOD</strong><br>
407Steganography: embed data to identify and track IP/credentials.</p>
408<br>
409<p>
410A legal strong-arm strategy, freely give data but track its distribution.</p>
411<br>
412<p>
413Useful to determine ToS violations.</p>
414<br>
415<p>
416Poison the well! Leave fake data buried deep within the dataset.</p>
417</section><section   class="vertical-slide">
418<p>
419<h3>Image steganography</h3>Hide data in the EXIF header (obvious place, easy to remove), <a href="http://www.sno.phy.queensu.ca/~phil/exiftool/">ExifTool</a></p>
420<figure>
421    <a href="images/honeypot_examples/panda.jpg">
422    <img class="" src="images/honeypot_examples/panda.jpg" >
423    </a>
424    <figcaption><a href="https://flic.kr/p/9eopJm">Kevin Dooley, Flickr</a></figcaption>
425    </figure>
426<br>
427<pre><code class=python>$ identify -verbose panda.jpg 
428
429 Image: panda.jpg
430  Format: JPEG (Joint Photographic Experts Group JFIF format)
431  ...
432  Properties:
433    date:create: 2016-01-10T11:58:10-05:00
434    exif:ApertureValue: 327680/65536
435    exif:ColorSpace: 1
436    exif:DateTime: 2009:08:01 08:59:44
437    exif:DateTimeOriginal: 2009:07:24 04:17:22
438   ...</code></pre>
439</section><section   class="vertical-slide">
440<p>
441<h3>Image steganography</h3>Map post-filter md5sum to user data (not resistant to image changes).<br>Impossible for user to know what is being stored!</p>
442<br>
443<pre><code class=python>import numpy as np
444from scipy.ndimage import imread
445from scipy.misc import imsave
446
447jpg = imread(&quot;panda.jpg&quot;)
448idx = np.random.uniform(size=jpg.shape) &lt; 0.001
449jpg[idx] += np.random.uniform(-2,2, size=idx.sum()).astype(np.uint8)
450jpg[jpg&lt;0] = 0
451jpg[jpg&gt;255] = 255
452imsave(&quot;panda_new.jpg&quot;, jpg)
453# Test on command line
454# $ md5sum *.jpg
455# bd1a44ba2111eb675e78935d4d5cc186  panda.jpg
456# 672c6dbf03828ea50a70bc81e19bfd69  panda_new.jpg</code></pre>
457<br>
458<figure>
459    <a href="images/honeypot_examples/panda.jpg">
460    <img class="" src="images/honeypot_examples/panda.jpg" >
461    </a>
462    
463    </figure>
464<figure>
465    <a href="images/honeypot_examples/panda_new.jpg">
466    <img class="" src="images/honeypot_examples/panda_new.jpg" >
467    </a>
468    
469    </figure>
470</section><section   class="vertical-slide">
471<p>
472<h3>General steganography</h3></p>
473<br>
474<p>
475Works for any lossy format (mp3, gif, etc...)<br>For tabular data, hide identification in NULL fields that can be easily removed.<br>Perturb date-times by seconds in data records and save the offset.</p>
476<br>
477<p>
478<h3>Honeypots</h3></p>
479<br>
480<p>
481If a bot or persistent downloader is identified, feed them faulty data.<br>Continually degrade image quality sent as function of DL's.<br>Remove rows, or return records not found with increasing frequency.</p>
482</section><section   class="vertical-slide">
483<p>
484<div class="whitehat"><h2><code>WhiteHat8</code>: Honeypots & Steganography </div></h2></p>
485<br>
486<p>
487Download data multiple times from different origins.</p>
488<br>
489<p>
490Run <code>diff</code> commands to suss out data that changes by IP and user.</p>
491<br>
492<p>
493Sanitize data by rejecting fields and entries that change with alternative DLs.</p>
494<br>
495<p>
496Modify image to remove steganography (apply same trick twice!)</p>
497</section>
498</section><section class="vertical-stack">
499<section id="Hack9"  class="vertical-slide">
500<p>
501<h2><code>BlackHat9</code>: Remove markup metadata</h2>implementation <strong>HARD</strong> : defense <strong>REASONABLE</strong> : hack-level <strong>CORPORATE</strong></p>
502<br>
503<p>
504<h3>Two ways:</h3><h4>1. Break the standard UX design.</h4><h4>2. Remove proper HTML/CSS markup.</h4></p>
505<br>
506<p>
507Organized webpage = Organized data = Easy rip</p>
508</section><section   class="vertical-slide">
509<p>
510Eschew all user design and layer components dynamically.<br>Example: <a href="http://arngren.net/">http://arngren.net/</a></p>
511<figure>
512    <a href="images/badUX_arngren.png">
513    <img class="" src="images/badUX_arngren.png" >
514    </a>
515    
516    </figure>
517</section><section   class="vertical-slide">
518<p>
519Remove markup. You can't rip what you can't see.</p>
520<br>
521<pre><code class=python>&lt;div class=&quot;author&quot;&gt;
522    &lt;div class=&quot;firstname&quot;&gt;Preston &lt;/div&gt;
523    &lt;div class=&quot;lastname&quot;&gt; Garvey  &lt;/div&gt;
524&lt;div&gt;
525
526&lt;div class=&quot;author&quot;&gt;
527    &lt;div class=&quot;firstname&quot;&gt;Piper &lt;/div&gt; 
528    &lt;div class=&quot;lastname&quot;&gt; Wright  &lt;/div&gt;
529&lt;div&gt;</code></pre>
530<br>
531<pre><code class=python>&lt;!-- Remove all class and id labels, like this --&gt; 
532&lt;div style=&quot;font-weight: bold;&quot;&gt;
533Preston Garvey &lt;/br&gt;
534Piper Wright
535&lt;/div&gt;</code></pre>
536</section><section   class="vertical-slide">
537<p>
538<div class="whitehat"><h2><code>WhiteHat9</code>: Remove markup</div></h2></p>
539<br>
540<p>
541Rare in the wild as this makes web development a nightmare.<br><em>often found when dev's use lazy CMS...</em></p>
542<br>
543<p>
544Removing meta data slows users down, but syntax rules can be written per item:</p>
545<pre><code class=python>html = '''
546&lt;div style=&quot;font-weight: bold;&quot;&gt;
547Preston Garvey &lt;/br&gt;
548Piper Wright
549&lt;/div&gt;'''
550
551import bs4
552soup = bs4.BeautifulSoup(html,'lxml')
553text = soup.div.text
554names = text.strip().split('\n')
555
556keys = &quot;firstname&quot;, &quot;lastname&quot;
557data = [dict(zip(keys,x.split())) for x in names]
558
559print data
560# [{'lastname': u'Garvey', 'firstname': u'Preston'}, {'lastname': u'Wright', 'firstname': u'Piper'}]</code></pre>
561</section>
562</section><section class="vertical-stack">
563<section id="Hack10"  class="vertical-slide">
564<p>
565<h2><code>BlackHat10</code>: HTML obfuscation</h2>implementation <strong>EASY</strong> : defense <strong>REASONABLE</strong> : hack-level <strong>SCRIPT-KIDDIE</strong></p>
566<br>
567<p>
568Encode everything with HTML character codes and insert random benign HTML.</p>
569<br>
570<p>
571Start with this:</p>
572<pre><code class=python>This is a string of text</code></pre>
573<p>
574Encode to this:</p>
575<pre><code class=python>&amp;#84;&amp;#104;&amp;#105;&amp;#115;&amp;#32;&amp;#105;&amp;#115;&amp;#32;&amp;#97;&lt;u&gt;&lt;/u&gt;&amp;#32;&amp;#115;
576&lt;i&gt;&lt;/i&gt;&amp;#116;&amp;#114;&amp;#105;&lt;u&gt;&lt;/u&gt;&amp;#110;&amp;#103;&lt;i&gt;&lt;/i&gt;&amp;#32;&lt;u&gt;&lt;/u&gt;&amp;#111;&amp;#102;&amp;#32;&amp;#116;&amp;#101;&amp;#120;&amp;#116;</code></pre>
577<p>
578'View Source' shows this:</p>
579<pre><code class=python>&lt;p&gt;&amp;#84;&amp;#104;&amp;#105;&amp;#115;&amp;#32;&amp;#112;&amp;#97;&amp;#103;&amp;#101;
580&amp;#32;&amp;#105;&amp;#115;&lt;i&gt;&lt;/i&gt;&amp;#32;&lt;u&gt;&lt;/u&gt;&amp;#109;&amp;#101;&amp;#97;&amp;#110;&lt;b&gt;&lt;/b&gt;</code></pre>
581</section><section   class="vertical-slide">
582<p>
583<div class="whitehat"><h2><code>WhiteHat10</code>: HTML obfuscation </div></h2></p>
584<br>
585<p>
586Use the <a href="http://www.seleniumhq.org/projects/webdriver/">Selenium Web Driver</a></p>
587<br>
588<ol><li>1. Create a headless web browser</li>
589<li>2. Open the page</li>
590<li>3. Take a screenshot of the page</li>
591<li>4. Use OCR to extract the text from the screenshot</li></ol>
592<p>
593<em>or</em></p>
594<ol><li>1. Capture the entire page (curl, etc.)</li>
595<li>2. Decode the HTML characters using <a href="http://www.crummy.c
595om/software/BeautifulSoup/">BeautifulSoup4</a></li></ol>
596</section>
597</section><section class="vertical-stack">
598<section id="Hack11"  class="vertical-slide">
599<p>
600<h2><code>BlackHat11</code>: Serving HTML as PDF</h2>implementation <strong>HARD</strong> : defense <strong>STRONG</strong> : hack-level <strong>CORPORATE</strong></p>
601<br>
602<p>
603<h3>Convert all Text to PDF</h3>Use <a href="http://phantomjs.org/">PhantomJS</a>, <a href="http://wkhtmltopdf.org/">Wkhtmltopdf</a> or <a href="http://pdfkit.org/">PDFKit</a> (node.js)</p>
604<br>
605<p>
606Eschew style conventions and use multi-columns!</p>
607</section><section   class="vertical-slide">
608<p>
609<div class="whitehat"><h2><code>WhiteHat11</code>: Serving HTML as PDF </div></h2></p>
610<br>
611<p>
612Use OCR to extract text and images from the text<br><em>or</em><br><a href="http://tabula.technology/">Tabula</a> to extract tabular data</p>
613</section>
614</section><section class="vertical-stack">
615<section id="Hack12"  class="vertical-slide">
616<p>
617<h2><code>BlackHat12</code>: Text remapping</h2>implementation <strong>WTF</strong> : defense <strong>RIDICULOUS</strong> : hack-level <strong>HOLLYWOOD</strong></p>
618<br>
619<p>
620Alter text from visual display:</p>
621<br>
622<p>
623<h3>Javascript</h3><h3>Hidden spans</h3><h3>Font remapping</h3></p>
624</section><section   class="vertical-slide">
625<p>
626<h2>Javascript text manipulation</h2></p>
627<br>
628<p>
629Alter the text as it is copied. JSfiddle <a href="http://jsfiddle.net/jp6nhmxf/">example</a></p>
630<br>
631<pre><code class=python>function addLink() {
632    //Get the selected text and append the extra info
633    var selection = window.getSelection(),
634        pagelink = '&lt;br /&gt;&lt;br /&gt; Read more at: ' + document.location.href,
635        copytext = selection + pagelink,
636        newdiv = document.createElement('div');
637
638    //hide the newly created container
639    newdiv.style.position = 'absolute';
640    newdiv.style.left = '-99999px';
641
642    //insert the container, fill it with the extended text, and define the new selection
643    document.body.appendChild(newdiv);
644    newdiv.innerHTML = copytext;
645    selection.selectAllChildren(newdiv);
646
647    window.setTimeout(function () {
648        document.body.removeChild(newdiv);
649    }, 100);
650}
651document.addEventListener('copy', addLink);</code></pre>
652</section><section   class="vertical-slide">
653<p>
654<h2>Hidden spans</h2></p>
655<br>
656<p>
657simple text below right?<h2><p class="codeblock">T<span style="position: absolute; left: -5000px; top: -5000px">gCRT3Qg3</span>R<span style="position: absolute; left: -5000px; top: -5000px">T7SQNdsF</span>A<span style="position: absolute; left: -5000px; top: -5000px">TBsh8T3T</span>V<span style="position: absolute; left: -5000px; top: -5000px">WKaKeTMg</span>I<span style="position: absolute; left: -5000px; top: -5000px">ayRwzhur</span>S<span style="position: absolute; left: -5000px; top: -5000px">tNVKkXZV</span></p></h2><div class="fragment">copy and paste transforms<br><code>TRAVIS</code> to <code>TgCRT3Qg3RT7SQNdsFATBsh8T3TVWKaKeTMgIayRwzhurS</code></p>
658<pre><code class=python> &lt;p class=&quot;codeblock&quot;&gt;
659   T
660   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;gCRT3Qg3&lt;/span&gt;
661   R
662   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;T7SQNdsF&lt;/span&gt;
663   A
664   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;TBsh8T3T&lt;/span&gt;
665   V
666   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;WKaKeTMg&lt;/span&gt;
667   I
668   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;ayRwzhur&lt;/span&gt;
669   S
670   &lt;span style=&quot;position: absolute; left: -100px; top: -100px&quot;&gt;tNVKkXZV&lt;/span&gt;
671 &lt;/p&gt;</code></pre>
672<br>
673<p>
674Any data payload can be inserted here (e.g. copyright claims, point of origin, etc...)<br></div></p>
675</section><section   class="vertical-slide">
676<p>
677<h2>Font remapping</h2></p>
678<br>
679<p>
680Render document to PDF and remap fonts <em>per document</em> for protected data.<h3>Example: <a href="images/font_remapping.pdf">font_remapping.pdf</a></h3><div class="fragment"></p>
681<br>
682<p>
683WTH? How does it work?</p>
684<blockquote>A PDF is a collection of symbols drawn on a page. Draw `c` here, draw `a` there, etc. A PDF reader only knows what a letter is because it maps to a specific character code in the font. Simply create a new font that lies about its mapping.</blockquote>
685<br>
686<p>
687Multiple fonts can be used to improve the "encryption" process,<br>one font per character gives a one-time pad!<br></div></p>
688</section><section   class="vertical-slide">
689<p>
690<div class="whitehat"><h2><code>WhiteHat12</code>
690: Text remapping </div></h2></p>
691<br>
692<p>
693For <strong>Javascript</strong> remapping use a headless browser. For <strong>hidden spans</strong>, learn and write custom rules to remove the offending page elements. For <strong>font remapping</strong>...<div class="fragment"><br><br>Throw money and humans at it: <a href="https://www.mturk.com/mturk/welcome">Mechanial Turk</a></p>
694<figure>
695    <a href="images/baby_rain.gif">
696    <img class="transparent_image" src="images/baby_rain.gif" height="420px">
697    </a>
698    
699    </figure>
700<figure>
701    <a href="images/monkey.gif">
702    <img class="transparent_image" src="images/monkey.gif" height="420px">
703    </a>
704    
705    </figure>
706<p>
707</div></p>
708</section>
709</section><section class="vertical-stack">
710<section   class="vertical-slide">
711<p>
712<h1>Thanks, you!</h1></p>
713<br>
714<p>
715Got any more Black Hat Hacks? Let us know!<h5><code>#blackhatdata</code> / <a href="https://twitter.com/metasemantic">@metasemantic</a> / <a href="https://twitter.com/rdempsey">@rdempsey</a></h5></p>
716</section>
717</section>
718    </div></div>
719
720    
720<script src="reveal.js/lib/js/head.min.js"></script>
720
721    
721<script src="reveal.js/js/reveal.js"></script>
721
722
723    
723<script>
724      // Full list of configuration options available here:
725      // https://github.com/hakimel/reveal.js#configuration
726      Reveal.initialize({
727      overview: true, height: 900, keyboard: {37: 'prev', 39: 'next'}, touch: true, hideAddressBar: true, rollingLinks: false, mouseWheel: true, controls: false, width: 1200, viewDistance: 3, rtl: false, progress: true, fragments: true, autoSlide: 0, autoSlideStoppable: true, backgroundtransition: "default", transitionSpeed: "default", slideNumber: true, center: true, embedded: true, transition: "default", maxScale: 1.2, minScale: 0.2, previewLinks: false, margin: 0.05, loop: false, history: true, 
728      
729      theme: Reveal.getQueryHash().theme, 
730      // available themes are in reveal.js/css/theme
731
732      // Optional libraries used to extend on reveal.js
733      dependencies: [ { src: 'reveal.js/lib/js/classList.js', condition: function() { 
734    return !document.body.classList; } },
735
736{ src: 'reveal.js/plugin/markdown/showdown.js', condition: function() { 
737    return !!document.querySelector( '[data-markdown]' ); } },
738
739{ src: 'reveal.js/plugin/markdown/markdown.js', condition: function() { 
740    return !!document.querySelector( '[data-markdown]' ); } },
741
742{ src: 'reveal.js/plugin/highlight/highlight.js', async: true, callback: 
743  function() { hljs.initHighlightingOnLoad(); } },
744
745{ src: 'reveal.js/plugin/zoom-js/zoom.js', async: true, condition: 
746  function() { return !!document.body.classList; } },
747 ]
748
749      });
750    </script>
750
751
752  </body>
753</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.