1<!DOCTYPE html> 2<html lang="en"> 3<head> 4<meta charset="UTF-8" /> 5<meta name="viewport" content="width=device-width, initial-scale=1" /> 6<title>Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS | cloudncode</title> 7<link rel="profile" href="http://gmpg.org/xfn/11" /> 8<link rel="pingback" href="https://cloudncode.blog/xmlrpc.php" /> 9<!--[if lt IE 9]>
10<script src="https://s2.wp.com/wp-content/themes/pub/expound/js/html5.js?m=1376965621i" type="text/javascript"></script>
10 11<![endif]--> 12 13<meta name='robots' content='max-image-preview:large' /> 14<meta name="google-site-verification" content="Pq8KitGCEw84V8ljNgiA-qSr6YraJ2vwWb4KUdK_Y6Y" /> 15<meta name="msvalidate.01" content="2E7FC93632C2882390CB6894C1657D64" /> 16<meta name="p:domain_verify" content="92b724c934777f301a90d32a724a6a50" /> 17 18<!-- Async WordPress.com Remote Login -->
19<script id="wpcom_remote_login_js"> 20var wpcom_remote_login_extra_auth = ''; 21function wpcom_remote_login_remove_dom_node_id( element_id ) { 22 var dom_node = document.getElementById( element_id ); 23 if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); } 24} 25function wpcom_remote_login_remove_dom_node_classes( class_name ) { 26 var dom_nodes = document.querySelectorAll( '.' + class_name ); 27 for ( var i = 0; i < dom_nodes.length; i++ ) { 28 dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] ); 29 } 30} 31function wpcom_remote_login_final_cleanup() { 32 wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" ); 33 wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" ); 34 wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" ); 35 wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" ); 36 wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" ); 37 wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" ); 38} 39 40// Watch for messages back from the remote login 41window.addEventListener( "message", function( e ) { 42 if ( e.origin === "https://r-login.wordpress.com" ) { 43 var data = {}; 44 try { 45 data = JSON.parse( e.data ); 46 } catch( e ) { 47 wpcom_remote_login_final_cleanup(); 48 return; 49 } 50 51 if ( data.msg === 'LOGIN' ) { 52 // Clean up the login check iframe 53 wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" ); 54 55 var id_regex = new RegExp( /^[0-9]+$/ ); 56 var token_regex = new RegExp( /^.*|.*|.*$/ ); 57 if ( 58 token_regex.test( data.token ) 59 && id_regex.test( data.wpcomid ) 60 ) { 61 // We have everything we need to ask for a login 62 var script = document.createElement( "script" ); 63 script.setAttribute( "id", "wpcom_remote_login_validate" ); 64 script.src = '/remote-login.php?wpcom_remote_login=validate' 65 + '&wpcomid=' + data.wpcomid 66 + '&token=' + encodeURIComponent( data.token ) 67 + '&host=' + window.location.protocol 68 + '//' + window.location.hostname 69 + '&postid=289' 70 + '&is_singular=1'; 71 document.body.appendChild( script ); 72 } 73 74 return; 75 } 76 77 // Safari ITP, not logged in, so redirect 78 if ( data.msg === 'LOGIN-REDIRECT' ) { 79 window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href; 80 return; 81 } 82 83 // Safari ITP, storage access failed, remove the request 84 if ( data.msg === 'LOGIN-REMOVE' ) { 85 var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }'; 86 var style_zap = document.createElement( 'style' ); 87 style_zap.type = 'text/css'; 88 style_zap.appendChild( document.createTextNode( css_zap ) ); 89 document.body.appendChild( style_zap ); 90 91 var e = document.getElementById( 'wpcom_request_access_iframe' ); 92 e.parentNode.removeChild( e ); 93 94 document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000'; 95 96 return; 97 } 98 99 // Safari ITP 100 if ( data.msg === 'REQUEST_ACCESS' ) { 101 console.log( 'request access: safari' ); 102 103 // Check ITP iframe enable/disable knob 104 if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) { 105 return; 106 } 107 108 // If we are in a "private window" there is no ITP. 109 var private_window = false; 110 try { 111 var opendb = window.openDatabase( null, null, null, null ); 112 } catch( e ) { 113 private_window = true; 114 } 115 116 if ( private_window ) { 117 console.log( 'private window' ); 118 return; 119 } 120 121 var iframe = document.createElement( 'iframe' ); 122 iframe.id = 'wpcom_request_access_iframe'; 123 iframe.setAttribute( 'scrolling', 'no' ); 124 iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' ); 125 iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid ); 126 127 var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } '; 128 129 var style = document.createElement( 'style' ); 130 style.type = 'text/css'; 131 style.id = 'wpcom_request_access_styles'; 132 style.appendChild( document.createTextNode( css ) ); 133 document.body.appendChild( style ); 134 135 document.body.appendChild( iframe ); 136 } 137 138 if ( data.msg === 'DONE' ) { 139 wpcom_remote_login_final_cleanup(); 140 } 141 } 142}, false ); 143 144// Inject the remote login iframe after the page has had a chance to load 145// more critical resources 146window.addEventListener( "DOMContentLoaded", function( e ) { 147 var iframe = document.createElement( "iframe" );
148 iframe.style.display = "none"; 149 iframe.setAttribute( "scrolling", "no" ); 150 iframe.setAttribute( "id", "wpcom_remote_login_key" ); 151 iframe.src = "https://r-login.wordpress.com/remote-login.php" 152 + "?wpcom_remote_login=key" 153 + "&origin=aHR0cHM6Ly9jbG91ZG5jb2RlLmJsb2c%3D" 154 + "&wpcomid=113813661" 155 + "&time=" + Math.floor( Date.now() / 1000 ); 156 document.body.appendChild( iframe ); 157}, false ); 158</script>
158 159<link rel='dns-prefetch' href='//s1.wp.com' /> 160<link rel='dns-prefetch' href='//s2.wp.com' /> 161<link rel='dns-prefetch' href='//widgets.wp.com' /> 162<link rel='dns-prefetch' href='//wordpress.com' /> 163<link rel='dns-prefetch' href='//s0.wp.com' /> 164<link rel='dns-prefetch' href='//af.pubmine.com' /> 165<link rel="alternate" type="application/rss+xml" title="cloudncode » Feed" href="https://cloudncode.blog/feed/" /> 166<link rel="alternate" type="application/rss+xml" title="cloudncode » Comments Feed" href="https://cloudncode.blog/comments/feed/" /> 167<link rel="alternate" type="application/rss+xml" title="cloudncode » Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS Comments Feed" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/feed/" /> 168
168<script type="text/javascript"> 169 /* <![CDATA[ */ 170 function addLoadEvent(func) { 171 var oldonload = window.onload; 172 if (typeof window.onload != 'function') { 173 window.onload = func; 174 } else { 175 window.onload = function () { 176 oldonload(); 177 func(); 178 } 179 } 180 } 181 /* ]]> */ 182 </script>
182 183 <style id="wp-img-auto-sizes-contain-inline-css"> 184img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px} 185/*# sourceURL=wp-img-auto-sizes-contain-inline-css */ 186</style> 187<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??/wp-content/mu-plugins/widgets/eu-cookie-law/templates/style.css,/wp-content/mu-plugins/likes/jetpack-likes.css?m=1743883414j&cssminify=yes' type='text/css' media='all' /> 188<style id="wp-emoji-styles-inline-css"> 189 190 img.wp-smiley, img.emoji { 191 display: inline !important; 192 border: none !important; 193 box-shadow: none !important; 194 height: 1em !important; 195 width: 1em !important; 196 margin: 0 0.07em !important; 197 vertical-align: -0.1em !important; 198 background: none !important; 199 padding: 0 !important; 200 } 201/*# sourceURL=wp-emoji-styles-inline-css */ 202</style> 203<style id="wp-block-library-inline-css"> 204:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;
204--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}} 205 206/*# sourceURL=/wp-content/plugins/gutenberg-core/v24.0.0/build/styles/block-library/common.min.css */ 207</style> 208<style id="wp-block-library-inline-css-extra"> 209.has-text-align-justify { 210 text-align:justify; 211} 212.has-text-align-justify{text-align:justify;} 213/*# sourceURL=wp-block-library-inline-css */ 214</style> 215 216<style id="classic-theme-styles-inline-css"> 217.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0} 218/*# sourceURL=/wp-content/plugins/gutenberg-core/v24.0.0/build/styles/block-library/classic.min.css */ 219</style> 220 221<style id="coblocks-inline-format-inline-css"> 222.uppercase{text-transform:uppercase} 223/*# sourceURL=coblocks-inline-format-inline-css */ 224</style> 225<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??/wp-content/mu-plugins/core-compat/wp-mediaelement.css,/wp-content/mu-plugins/wpcom-bbpress-premium-themes.css?m=1432920480j&cssminify=yes' type='text/css' media='all' /> 226<style id="global-styles-inline-css"> 227:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,
2270) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}
227.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;} 228/*# sourceURL=global-styles-inline-css */ 229</style> 230 231<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.css?m=1785880900i&cssminify=yes' type='text/css' media='all' /> 232<link rel='stylesheet' id='verbum-gutenberg-css-css' href='https://widgets.wp.com/verbum-block-editor/block-editor.css?ver=1771405207' media='all' /> 233<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJx9jVEOwjAMQy9EicYQiA/EWbYSSqBNqjXV4PYEkGCAxJ/9EtswZueFFVkhVZdjDcQFvKRkyEU6o7nyReZGZkDAomTp8hLPw6RTj5isIdce8JKl8h6KXiP+PE7GB+yjBJPhsfy2/0IBxUXxnZLwh3GH2NFwj+7Stlkvm9WmbdvF6QaGbl0a&cssminify=yes' type='text/css' media='all' /> 234<style id="jetpack-global-styles-frontend-style-inline-css"> 235:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;} 236/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */ 237</style> 238<link crossorigin='anonymous' rel='stylesheet' id='all-css-18-1' href='/_static/??-eJyNjcsKAjEMRX/ImhmUURfip0inLW3GNCmmQfx7H7gRN+7ugcs5cGsuCPfEHaq5RpaRFZbUmw+XD4MaQ5VolBS0+GuKPsb7eyLndVBdwd+iM3KA2ZAiqAT05Eiy6Bf8KHtJ9RkvW8gks6fX4VSP426/mQ7TMA7LA38FSqA=&cssminify=yes' type='text/css' media='all' />
239<script id="wpcom-actionbar-placeholder-js-extra"> 240var actionbardata = {"siteID":"113813661","postID":"289","siteURL":"https://cloudncode.blog","xhrURL":"https://cloudncode.blog/wp-admin/admin-ajax.php","nonce":"ba836f7ed5","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&ver=20211021","shortlink":"https://wp.me/p7Hy8d-4F","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Copied to clipboard."}}; 241//# sourceURL=wpcom-actionbar-placeholder-js-extra 242</script>
vendor: 1 bytes, line 242
242
243<script id="jetpack-mu-wpcom-settings-js-before"> 244var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"}; 245//# sourceURL=jetpack-mu-wpcom-settings-js-before 246</script>
vendor: 1 bytes, line 246
246
247<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/js/rlt-proxy.js,/wp-content/blog-plugins/wordads-classes/js/cmp/v2/cmp-non-gdpr.js?m=1780567333j'></script>
vendor: 1 bytes, line 247
247
248<script id="rlt-proxy-js-after"> 249 rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} ); 250//# sourceURL=rlt-proxy-js-after 251</script>
251 252<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://cloudncode.wordpress.com/xmlrpc.php?rsd" /> 253<meta name="generator" content="WordPress.com" /> 254<link rel="canonical" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" /> 255<link rel='shortlink' href='https://wp.me/p7Hy8d-4F' /> 256<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&url=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&url=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&for=wpcom-auto-discovery" /> 257<!-- Jetpack Open Graph Tags --> 258<meta property="og:type" content="article" /> 259<meta property="og:title" content="Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS" /> 260<meta property="og:url" content="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" /> 261<meta property="og:description" content="This is the third article in the series on What should you log in an application and how to avoid having 24Ã7 support looking at them? In the previous article we talked about how to setup Elastic Sâ¦" /> 262<meta property="article:published_time" content="2017-02-06T12:56:42+00:00" /> 263<meta property="article:modified_time" content="2017-05-24T14:22:56+00:00" /> 264<meta property="og:site_name" content="cloudncode" /> 265<meta property="og:image" content="https://i0.wp.com/cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?fit=1200%2C788&ssl=1" /> 266<meta property="og:image:width" content="1200" /> 267<meta property="og:image:height" content="788" /> 268<meta property="og:image:alt" content="" /> 269<meta property="og:locale" content="en_US" /> 270<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" /> 271<meta name="twitter:creator" content="@maingified" /> 272<meta name="twitter:site" content="@maingified" /> 273<meta name="twitter:text:title" content="Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS" /> 274<meta name="twitter:image" content="https://i0.wp.com/cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?fit=1200%2C788&ssl=1&w=640" /> 275<meta name="twitter:card" content="summary_large_image" /> 276 277<!-- End Jetpack Open Graph Tags --> 278<link rel="shortcut icon" type="image/x-icon" href="https://s1.wp.com/i/favicon.ico?m=1713425267i" sizes="16x16 24x24 32x32 48x48" /> 279<link rel="icon" type="image/x-icon" href="https://s1.wp.com/i/favicon.ico?m=1713425267i" sizes="16x16 24x24 32x32 48x48" /> 280<link rel="apple-touch-icon" href="https://s2.wp.com/i/webclip.png?m=1713868326i" /> 281<link rel="search" type="application/opensearchdescription+xml" href="https://cloudncode.blog/osd.xml" title="cloudncode" /> 282<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" /> 283<meta name="description" content="This is the third article in the series on What should you log in an application and how to avoid having 24Ã7 support looking at them? In the previous article we talked about how to setup Elastic Search and Kibana, in this article weâll learn how to properly send logs from your application servers to…" />
284<script> 285var wa_client = {}; wa_client.cmd = []; wa_client.config = { 'blog_id': 113813661, 'blog_language': 'en', 'is_wordads': true, 'hosting_type': 1, 'afp_account_id': null, 'afp_host_id': 5038568878849053, 'theme': 'pub/expound', '_': { 'title': 'Advertisement', 'privacy_settings': 'Privacy Settings' }, 'formats': [ 'inline', 'belowpost', 'bottom_sticky', 'sidebar_sticky_right', 'sidebar', 'top', 'shortcode', 'gutenberg_rectangle', 'gutenberg_leaderboard', 'gutenberg_mobile_leaderboard', 'gutenberg_skyscraper' ] }; 286</script>
286 287
287<script type="text/javascript"> 288 289 window.doNotSellCallback = function() { 290 291 var linkElements = [ 292 'a[href="https://wordpress.com/?ref=footer_blog"]', 293 'a[href="https://wordpress.com/?ref=footer_website"]', 294 'a[href="https://wordpress.com/?ref=vertical_footer"]', 295 'a[href^="https://wordpress.com/?ref=footer_segment_"]', 296 ].join(','); 297 298 var dnsLink = document.createElement( 'a' ); 299 dnsLink.href = 'https://wordpress.com/advertising-program-optout/'; 300 dnsLink.classList.add( 'do-not-sell-link' ); 301 dnsLink.rel = 'nofollow'; 302 dnsLink.style.marginLeft = '0.5em'; 303 dnsLink.textContent = 'Do Not Sell or Share My Personal Information'; 304 305 var creditLinks = document.querySelectorAll( linkElements ); 306 307 if ( 0 === creditLinks.length ) { 308 return false; 309 } 310 311 Array.prototype.forEach.call( creditLinks, function( el ) { 312 el.insertAdjacentElement( 'afterend', dnsLink ); 313 }); 314 315 return true; 316 }; 317 318 </script>
318 319 <link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s1.wp.com/?custom-css=1&csblog=7Hy8d&cscache=6&csrev=9" /> 320
320<script type="text/javascript"> 321 window.google_analytics_uacct = "UA-52447-2"; 322</script>
322 323
324<script type="text/javascript"> 325 var _gaq = _gaq || []; 326 _gaq.push(['_setAccount', 'UA-52447-2']); 327 _gaq.push(['_gat._anonymizeIp']); 328 _gaq.push(['_setDomainName', 'none']); 329 _gaq.push(['_setAllowLinker', true]); 330 _gaq.push(['_initData']); 331 _gaq.push(['_trackPageview']); 332 333 (function() { 334 var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; 335 ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; 336 (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(ga); 337 })(); 338</script>
338 339<style id="jetpack-subscriptions-inline-css"> 340#subscribe-email input { 341 width: 95%; 342} 343 344.comment-subscription-form { 345 margin-bottom: 1em; 346} 347 348.comment-subscription-form .subscribe-label { 349 display: inline !important; 350} 351 352/* 353Text meant only for screen readers. 354Provides support for themes that do not bundle this CSS yet. 355@see https://make.wordpress.org/accessibility/2015/02/09/hiding-text-for-screen-readers-with-wordpress-core/ 356***********************************/ 357.screen-reader-text { 358 border: 0; 359 clip-path: inset(50%); 360 height: 1px; 361 margin: -1px; 362 overflow: hidden; 363 padding: 0; 364 position: absolute !important; 365 width: 1px; 366 overflow-wrap: normal !important; 367} 368 369/*# sourceURL=/wp-content/mu-plugins/jetpack-plugin/sun/_inc/build/subscriptions/subscriptions.min.css */ 370</style> 371 372</head> 373 374<body class="wp-singular post-template-default single single-post postid-289 single-format-standard custom-background wp-theme-pubexpound customizer-styles-applied jetpack-reblog-enabled"> 375<div id="page" class="hfeed site"> 376 <header id="masthead" class="site-header" role="banner"> 377 <div class="site-branding"> 378 <a href="https://cloudncode.blog/" class="wrapper-link"> 379 <div class="site-title-group"> 380 <h1 class="site-title"><a href="https://cloudncode.blog/" title="cloudncode" rel="home">cloudncode</a></h1> 381 <h2 class="site-description">Cloud Computing and the Code that Powers it</h2> 382 </div> 383 </a> 384 </div> 385 386 <nav id="site-navigation" class="navigation-main" role="navigation"> 387 <h1 class="menu-toggle">Menu</h1> 388 <div class="screen-reader-text skip-link"><a href="#content" title="Skip to content">Skip to content</a></div> 389 390 <div class="menu-primary-container"><ul id="menu-primary" class="menu"><li id="menu-item-12" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-12"><a href="/">Home</a></li> 391<li id="menu-item-420" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-420"><a href="https://cloudncode.blog/category/serverless-design/">Serverless Programming</a></li> 392<li id="menu-item-179" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-179"><a href="https://cloudncode.blog/category/cloud/">Cloud Related Articles</a></li> 393<li id="menu-item-180" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-180"><a href="https://cloudncode.blog/category/code/">Code Related Articles</a></li> 394<li id="menu-item-13" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-13"><a href="https://cloudncode.blog/about/">About</a></li> 395<li id="menu-item-14" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-14"><a href="https://cloudncode.blog/contact/">Contact</a></li> 396</ul></div> </nav><!-- #site-navigation --> 397 </header><!-- #masthead --> 398 399 <div id="main" class="site-main"> 400 401 <div id="primary" class="content-area"> 402 <div id="content" class="site-content" role="main"> 403 404 405 406<article id="post-289" class="post-289 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud category-code category-serverless-design tag-elasticsearch tag-elasticsearch-on-aws tag-getting-kinesis-firehose-to-put-data-in-elastic-search tag-getting-started-with-kinesis-in-aws tag-how-to-handle-terrabyte-level-log-data-in-aws tag-how-to-horizontally-scale-log-storage-on-aws tag-kinesis-firehose tag-using-aws-lambda-with-kinesis"> 407 <header class="entry-header"> 408 <h1 class="entry-title">Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS</h1> 409 410 <div class="entry-meta"> 411 <a class="author" rel="author" href="https://cloudncode.blog/author/maingi4/">Sumit Maingi</a> / <a class="entry-date" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/">February 6, 2017</a> </div><!-- .entry-meta --> 412 </header><!-- .entry-header --> 413 414 <div class="entry-content"> 415 <p>This is the third article in the series on <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/">What should you log in an application and how to avoid having 24Ã7 support looking at them?</a> In the previous article we talked about <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">how to setup Elastic Search and Kibana</a>, in this article weâll learn how to properly send logs from your application servers to Elastic search and do some processing in between and after we are done, weâll have a system ready which can handle terrabytes of log data with rare human intervention requirements.<span id="more-289"></span></p> 416<h2 id="so-what-is-this-kinesis-thingie">So⦠what is this Kinesis thingie?</h2> 417<p>Kinesis is an AWS data streaming service whose job is to deliver data reliably from one end to the other while adding extensions so as to make processing data easy and <em>scalable</em>. It is divided into two <em>main</em> sub-products:</p> 418<ol> 419<li><a href="https://aws.amazon.com/kinesis/streams/">
419Kinesis Streams</a>: Open ended data streams where you have data producers and consumers, you can define your own data consumers, this one does not scale on its own and you have to define the number of shards (capacity) you want from it beforehand.</li> 420<li><a href="https://aws.amazon.com/kinesis/firehose/">Kinesis Firehose</a>: This is same as above with the difference that it scales automatically and although data producers can be anything, data consumers are pre-defined, at the time of writing the article, valid consumers were:a. <a href="https://aws.amazon.com/s3/">Amazon S3</a>b. <a href="https://aws.amazon.com/elasticsearch-service/">Amazon Elastic Search service</a>c. <a href="https://aws.amazon.com/redshift/">Amazon Redshift</a>d. <a href="https://aws.amazon.com/kinesis/analytics/">Amazon Kinesis Analytics service</a></li> 421</ol> 422<p>Because we donât want to worry about scaling and are interested in putting the data in AWS ES service anyway, we are going to go with Kinesis Firehose. If there is still some confusion about what this is for, it will get cleared up as we try setting stuff up below.</p> 423<h2 id="why-kinesis-why-not-simply-send-stuff-to-elastic-search-directly">Why Kinesis? Why not simply send stuff to Elastic Search directly?</h2> 424<h3 id="pt1-because-logs-also-contain-blobs">Pt#1: Because logs also contain blobs</h3> 425<p>Typically logs contains exception logs, server events, application events (Trace, debug infoâs and request/responses) all these are necessary for good analysis of your data and is key to correctly and swiftly analyzing log data for diagnostic or analytical purposes. Now log lines (line = 1 entry of whatever type) which contains potentially huge chunks of unstructured data like request/responses which is never really searched upon and only looked at for diagnosing bugs, are actually increasing the packet size of what Elastic search is crunching across nodes. Even if the field is marked for no indexing (see here for <a href="https://www.elastic.co/guide/en/elasticsearch/guide/current/mapping-intro.html#_index_2">ES options on field indexing</a>) the data will still play a role in cluster health in the following scenarios:</p> 426<ol> 427<li>Shard redistribution (common) happens when you add a node to ES, or a node goes offline and replica shards get activated and the load gets redistributed across nodes for balancing, when the node comes back up the redistribution will begin again causing cascading failures (shudder) in case the size of the shards are too large</li> 428<li>Shard resizing (less often), happens when you change the number of shards for an index</li> 429<li>The Node coordinating the search result across various shards across different machines needs to collate large amounts of data size-wise in order to return the merged result, this is very resource intensive and may cause intermittent node outage, resulting in shard redistribution which again might cause the same thing due to large shard sizes, cascading failure is back!</li> 430</ol> 431<p>The only way to get away from the above stated problems would therefore be to throw GBâs of more RAM and storage coupled with double digit ethernet bandwidth with double the required nodes, if you shuddered at the thought of this statement then no, donât just store everything on ES.</p> 432<h3 id="pt2-because-lots-of-logs-are-unprocessed">Pt#2: Because lots of logs are unprocessed</h3> 433<p>Logs, a lot of times are unprocessed e.g. you might want to add geographical data (lat/long) to the user IP address that you collected (imagine heat maps in Kibana), server events which are just text lines and cannot be searched upon (you want to convert these to JSON with readable attributes before you shove it in ES), lots of other things. What you donât want to do is do these processings on your web servers. Web serverâs primary responsibility is to serve your customers the best and as fast as they can, doing these resource intensive work on each of your servers and subsequently making sure that all your services which make up your application are following the same standard is just not worth it.</p> 434<p>A good log data pipeline makes sure you have a scalable way to process these logs before they reach your log database so that your web servers are not effected while at the same time you have good searchable data that you can build amazing, actionable dashboards and alarms from.</p>
435<h3 id="hmm-ok-then-what-should-i-be-storing-in-es-now">Hmm.. ok.. then what should I be storing in ES now?</h3> 436<p>Elastic search as its name implies is brilliant at searching and even aggregating results across a humongous dataset while giving you near perfect horizontal scalability, use it to store fields that you search upon, aggregate upon. If you need to search for something in the request or response, you should be taking out that value and putting that in a separate field.</p> 437<h3 id="ok-ill-buy-that-where-do-you-suppose-i-put-unstructured-data-in-then">Ok, Iâll buy that, where do you suppose I put unstructured data in then?</h3> 438<p>Unstructured blobs (generally request/responses) should be put in storage optimized for large blob storage, which are essentially disks, in the AWS world, you have two options <a href="https://aws.amazon.com/s3/">S3</a> (http based file storage) and <a href="https://aws.amazon.com/efs/">EFS</a> (made for cloud, network drive). In this article, Iâll be using S3 as the blob storage mechanism, simply because its easier to integrate and has easy auto-archiving strategies.</p> 439<p>In this guide weâll send the request/response through Kinesis and just before stuff gets into Elastic search weâll have Lambda functions massaging the log data, take out the blobs, put them into S3 and replace the log lines with clickable S3 links instead.</p> 440<h3 id="are-there-any-limitations-to-kinesis">Are there any limitations to Kinesis?</h3> 441<p>While I would love to say that Kinesis is the anti-Gandalf and anything can pass through, it does have the following limitations:</p> 442<ol> 443<li>The maximum size of a record sent to Firehose, before base64-encoding, is 1000 KB (you can compress your request/response to meet this requirement)</li> 444<li>The PutRecordBatch (bulk insert into the stream) operation can take up to 500 records per call or 4 MB per call, whichever is smaller. This limit cannot be changed</li> 445<li>Data is buffered for a minimum of 60 seconds or 1 MB (whichever is hit first), which basically means if you donât have enough data going through the stream you might have to wait for a minute for your records to show up, this might be irritating in development environments, for production this is fine</li> 446</ol> 447<p>There are other AWS account related limitations which are worth noting, here is a <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">link to AWS documentation</a> for the complete set of Kinesis limitations, quite a few of these limitations can be increased by a simple support ticket to AWS via the console.</p> 448<h2 id="enough-talk-lets-setup-the-log-data-pipeline-already">Enough talk, Letâs setup the log data pipeline already!</h2> 449<h3 id="setting-up-elastic-search">Setting up Elastic search</h3> 450<p>I have the elastic search (5.1) already setup using AWS Elastic search service, <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">see this article</a> where I take you step by step in how to do that.</p> 451<h3 id="setting-up-a-index-template-for-time-based-log-indexes">Setting up a index template for time-based log indexes</h3> 452<p>Now if you went through the above mentioned article to setup your ES you already know that you <em>should</em> have time-based indexes for logging in ES, however, in order to control the behavior of these auto-generated time based indexes you can put index templates in ES so that whenever a new index is created for the day/week/month etc it automatically inherits the traits from the template that we define.</p> 453<p>We will define 4 shards per index and one replica since in the test setup we have only one elastic search node, you might want to have at least 2 replicaâs in production.</p> 454<pre class="brush: jscript; title: ; notranslate" title=""> 455PUT _template/active-logs 456{ 457"template": "active-logs-*", 458"settings": { 459"number_of_shards": 4, 460"number_of_replicas": 1 461} 462} 463</pre> 464<p><em>There is a maintenance overhead for shards so you donât want too many shards for an index however you also donât want gargantuan shards which w
464ould overload a nodeâs resources while aggregating/transferring/replicating etc.</em></p> 465<h3 id="getting-lambda-methods-to-intercept-the-log-data-and-slicing-blobs-into-s3">Getting Lambda methods to intercept the log data and slicing blobs into S3</h3> 466<p>We will transform and put the request and response in a S3 bucket as discussed before, create the lambda function exactly as I demonstrated <a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/">in this post</a> and swap the <code>program.cs</code> file with this:</p> 467<pre class="brush: csharp; title: ; notranslate" title=""> 468using System; 469using System.Collections.Generic; 470using System.IO; 471using System.Threading.Tasks; 472using Amazon.S3; 473using Amazon.S3.Model; 474 475namespace CSharpLambdaFunction 476{ 477public class LambdaHandler 478{ 479public static void Main() 480{ 481Task.Run(async () => 482{ 483Stream result; 484using (var stream = new FileStream($"{Directory.GetCurrentDirectory()}\\testData.json", FileMode.Open)) 485{ 486result = await new LambdaHandler().LogHandler(stream); 487} 488using (var reader = new StreamReader(result)) 489{ 490var records = reader.ReadToEnd(); 491 492Console.WriteLine(records); 493} 494}).GetAwaiter().GetResult(); 495} 496 497public async Task LogHandler(Stream inputStream) 498{ 499LogEntry entry; 500FirehoseInput input; 501using (var reader = new StreamReader(inputStream)) 502{ 503input = Deserialize(reader); 504} 505 506foreach (var record in input.records) 507{ 508record.result = "Ok"; 509using (var memStream = new MemoryStream(Convert.FromBase64String(record.data))) 510{ 511using (var reader = new StreamReader(memStream)) 512{ 513entry = Deserialize(reader); 514} 515} 516 517var reqTask = SendToS3AndGetLink(entry.Request); 518var resTask = SendToS3AndGetLink(entry.Response); 519 520entry.Request = await reqTask; 521entry.Response = await resTask; 522 523var data = ConvertToStream(entry); 524 525record.data = Convert.ToBase64String(data.ToArray()); 526} 527 528return ConvertToStream(input); 529} 530 531private T Deserialize(TextReader reader) 532{ 533var serializer = new Newtonsoft.Json.JsonSerializer(); 534 535return (T)serializer.Deserialize(reader, typeof(T)); 536} 537 538private MemoryStream ConvertToStream(T logEntry) 539{ 540var serializer = new Newtonsoft.Json.JsonSerializer(); 541 542var memStream = new MemoryStream(); 543var writer = new StreamWriter(memStream); 544serializer.Serialize(writer, logEntry); 545 546writer.Flush(); 547 548memStream.Position = 0; 549 550return memStream; 551} 552 553private Stream ConvertToStream(string value) 554{ 555var memStream = new MemoryStream(); 556var writer = new StreamWriter(memStream); 557 558writer.Write(value); 559writer.Flush(); 560 561return memStream; 562} 563 564private async Task SendToS3AndGetLink(string value) 565{ 566var s3 = new AmazonS3Client(Amazon.RegionEndpoint.USWest2); //define your own region here 567 568var putRequest = new PutObjectRequest(); 569 570var key = Guid.NewGuid().ToString().Replace("-", string.Empty); 571putRequest.BucketName = GetBucketName(); 572putRequest.Key = key; 573putRequest.InputStream = ConvertToStream(value); 574putRequest.ContentType = "application/json"; 575 576var response = await s3.PutObjectAsync(putRequest); 577 578var urlRequest = new GetPreSignedUrlRequest(); 579urlRequest.BucketName = GetBucketName(); 580urlRequest.Expires = DateTime.UtcNow.AddYears(2); 581urlRequest.Key = key; 582 583return s3.GetPreSignedURL(urlRequest); 584} 585 586private string GetBucketName() 587{ 588return "cloudncode-logs"; 589} 590} 591 592public class FirehoseInput 593{ 594public string invocationId { get; set; } 595public string deliveryStreamArn { get; set; } 596public string region { get; set; } 597public List records { get; set; } 598} 599public class FirehoseRecord 600{ 601public string recordId { get; set; } 602public string result { get; set; } 603public string data { get; set; } 604} 605 606public class LogEntry 607{ 608public string RequestId { get; set; } 609public string SessionId { get; set; } 610public string Timestamp { get; set; } 611public string UserId { get; set; } 612public string ServerIp { get; set; } 613public string Message { get; set; } 614public string Level { get; set; } 615public string Category { get; set; } 616public string Request { get; set; } 617public string Response { get; set; } 618} 619} 620</pre> 621<p>if you donât want to learn how to do that right now, you can also choose to <a href="https://cloudncode.blog/wp-content/uploads/2017/02/logprocessor-lambda_kinesis.zip">download the source code</a> directly from here, but you will need Visual Studio Code to compile and run it. If you are simply downloading the source code, go the article I mentioned above and start from <em>“Step 6: Add AWS users and roles”</em>.</p> 622<p>Note that the source code here is slightly different, the Lambda article that I wrote talks about a general Lambda function, however, Kinesis firehose gives a specific input and expects a specific output for the transformation to work. For our general knowledge, here is what Kinesis firehose gives as input to our Lambda function:</p> 623<pre class="brush: jscript; title: ; notranslate" title=""> 624{ 625"invocationId" : "cd6eb022-9528-42c6-82fb-e5e969ea7962", 626"deliveryStreamArn" : "arn:aws:firehose:us-west-2:522390xxx:deliverystream/logstream", 627"region" : "us-west-2", 628"records" : [{ 629"recordId" : "49570122502395591010539641575388588252247858295703863298", 630"approximateArrivalTimestamp" : 1486135421633, 631"data" : "" 632} 633] 634} 635</pre> 636<p>And here is the output that it expects:</p> 637<pre class="brush: jscript; title: ; notranslate" title=""> 638{ 639"invocationId": "cd6eb022-9528-42c6-82fb-e5e969ea7962", 640"deliveryStreamArn": "arn:aws:firehose:us-west-2:522390xxx:deliverystream/logstream", 641"region": "us-west-2", 642"records": [ 643{ 644"recordId": "49570122502395591010539641575388588252247858295703863298", 645"result": "Ok", //Valid options are "Ok", "Dropped" & "ProcessingFailed" 646"data": "" 647} 648] 649} 650</pre> 651<p>In the <code>record.result</code> field in the output above stating <code>Dropped</code> marks the result to be successful and indicates that the stream should stop further processing of the object and not send it any further.</p> 652<h3 id="visualizing-kinesis-data-flow">Visualizing Kinesis Data Flow</h3> 653<p>Before we move further and create the actual data stream, letâs make sure we have a picture in mind on how the Kinesis data stream flows from the origin (app servers/devices) to the destination which is Elasticsearch in our case.</p> 654<p>Consider the diagram below:</p> 655<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?w=1700" alt="enter image description here" /></p> 656<ol> 657<li>Kinesis receives data from your app server or directly via a smartphone app</li> 658<li>The data stream invokes a Lambda function which takes out request and response and decompresses the incoming log if it was compressed from the origin (in our case for simplicityâs sake, we wonât compress it) and puts them in an S3 bucket and further proceeds to replace the same with URLâs which the Kibana users can click on later to view the req/res in their browsers</li> 659<li>Lambda then returns the transformed log back to Kinesis which then puts them into the AWS Elasticsearch service</li> 660<li>In case any logs fail and all retries fail (retries are configurable), the failed logs are put in a configurable S3 bucket which can then be processed seperately, note that you can also dump every log into an S3 bucket along with the Elasticsearch service, however unless you mind not getting a fat bill from AWS I would dissuade against that</li> 661<li>Logs can then be viewed by the user using Kibana</li> 662</ol> 663<h3 id="setting-up-kinesis-firehose">Setting up Kinesis Firehose</h3> 664<p>Go to AWS Kinesis service page and then select Kinesis Firehose as the type of data stream, proceed to click on âCreate Delivery Streamâ as shown below:</p> 665<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-1.png?w=1700" alt="Create a delivery stream" /></p> 666<p>We want the data stream (Kinesis firehose) to send the data to our already setup AWS Elastic Search service, follow these steps:</p> 667<ol> 668<li>Choose Amazon Elasticsearch Service in the destination dropdown</li> 669<li>Give your stream a name, remember it will contain log data</li> 670<li>Choose your ES domain and type in <code>active-logs</code> in the index textbox, this name must correspond to the temaplate pattern that we had setup earlier</li> 671<li>Choose index rotation to OneDay, you can choose something different here as well depending upon the amount of traffic that you normally get in a day. A day is a safe choice if you are confused</li> 672<li>Put in <code>applicationEvent</code> in the type, this is the ES type in the index that will hold the documents, we want this stream to put in application events for now</li> 673<li>Setup to choose to backup your failed documents in S3, you can choose the safer option to backup everything as well, remember S3 charges per PUT request, this is more of a <em>what you can afford to do</em> kind of a question</li> 674<li>Create a new S3 bucket here, the dropdown will guide on doing the same</li> 675<li>Click on <code>Next</code></li> 676</ol> 677<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-2.png?w=1700" alt="Creating Kinesis Firehose step 1" /></p> 678<p>In this step, weâll configure our data stream to use the Lambda function that we had created earlier which will take away the request and responses and replace them with Urlâs, simple steps here, just refer the image in case you get confused.<
678/p> 679<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-3.png?w=1700" alt="Creating Kinesis Firehose step 2" /></p> 680<ol> 681<li>Setup the <code>buffer size</code> to be <code>5MB</code> and <code>interval</code> to be <code>60 seconds</code>, basically Kinesis will collect and send in logs together to ES for efficiency sake and will do that as soon as any one of the two conditions are fulfilled, i.e. either the collected documents reach 5MB in size or 60 seconds have elapsed since the documents had started to get collected.</li> 682<li>Since the backup S3 bucket only contains failures, you donât need compression here, you should compress docs if you are backing up everything to S3 though</li> 683<li>Leave the S3 data unencrypted, I prefer controlling access to S3 instead, makes life simpler</li> 684<li><code>Keep</code> the <code>error logging enabled</code>, it will help us in diagnosing any issues with our data stream</li> 685<li>Create an <code>IAM role</code> for your stream which will give it access to send backups to S3, invoke the Lambda function and put documents into ES, when you select âCreate a new roleâ from the dropdown it guides you in creating the role with just enough permissions to get the stream to work</li> 686<li>Click on <code>Next</code><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-4.png?w=1700" alt="Creating Kinesis Firehose step 3" /></li> 687</ol> 688<h3 id="sending-data-to-kinesis">Sending data to Kinesis</h3> 689<p>Logs can be sent to Kinesis via a Kinesis agent or via the AWS SDK, the agent although a pretty good option is currently available only for Amazon Linux AMI with version 2015.09 or later, or Red Hat Enterprise Linux version 7 or later. The agent basically picks up files from your server and transmits them to Kinesis completely decoupling your app from Kinesis. To know more about this option go to this <a href="http://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html">AWS documentation</a>.</p> 690<p>If you will be sending data via the AWS SDK from either a app server or a end-user device such as a smartphone you can check out the <a href="http://docs.aws.amazon.com/firehose/latest/dev/writing-with-sdk.html">AWS documentation</a> link for the same or you can download a simple utility that I built to quickly test things out, here is the <a href="https://cloudncode.blog/wp-content/uploads/2017/02/logprocessor-sampleloggenerator.zip">source code</a> for a sample tool that I made in dotnet core using Visual Studio Code, when you have the code opened, you can just change the <code>testData.json</code> file and put in whatever log request you want and run it. As always, change the region name in the code and make sure your AWS credentials file (<code>C:\Users\\.aws\credentials</code> for windows) contains valid credentials with permissions to send data to Kinesis. If you donât have these credentials configured, you can learn how to do that from <a href="http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-quick-configuration">this link at AWS</a> site which covers other operating systems apart from windows as well.</p> 691<h3 id="seeing-the-results-on-elastic-search-through-kibana">Seeing the results on Elastic search (through Kibana)</h3> 692<p>At this point you should have:</p> 693<ol> 694<li>A running AWS Elastic search service with templates added for our indexes which will be created via Kinesis firehose</li> 695<li>A Lambda function which we will use to transform our logs and related S3 bucket where the request and responses will be moved into</li> 696<li>Kinesis Firehose all setup to receive data from the servers/devices</li> 697<li>Some test data inserted into Elastic search</li> 698</ol> 699<p>Now we will setup Kibana with a simple table example so that users can actually look at the logs that go into the system for analysis and action.</p> 700<p>Open Kibana, as a refresher you can find the URL from the same from your Elasticsearch AWS console page, refer image below:</p> 701<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-4.png?w=768" alt="AWS Elasticsearch console" /></p> 702<p>We need to add our index pattern into Kibana so that it knows that it needs to read the same, go to the management tab in Kibana and click on <code>Index Patterns</code></p> 703<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-1.png?w=1700" alt="Kibana add index" /></p> 704<p>Click on <code>Add New</code></p> 705<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-2.png?w=700" alt="Kibana add index 2" /></p> 706<p>Put in <code>active-logs-*</code> in the index pattern box, The â*â here denotes that as new indexes are created based on time they will all be included in the search that you make, then proceed to click on <code>Create</code>.</p> 707<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-3.png?w=1700" alt="Kibana add index 3" /></p> 708<p>After the index is created, set it as default, this is optional of course, but as I had another index created while I wrote the â<a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">How to setup Elasticsearch</a>â post, I didnât want that to be my default.</p> 709<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-4.png?w=1700" alt="Kibana set index as default" /></p> 710<p>Go to the <code>Discover</code> tab and change your index to the correct one if not already (refer image), in case you donât see your test data as shown in the image try changing the time from the top-right corner in case the data was added in the past as in my case, click on <code>Save</code> and fill in name for the query.</p> 711<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-5.png?w=1700" alt="Kibana creating query (Saved Search)" /></p> 712<p>Now we will add a dashboard where we will save add the data table which users can later come and look at.</p> 713<p>Go to the <code>Dashboard</code> tab, click on <code>Add</code> and go to the <code>Saved Search</code> tab in the <code>Add Panels</code> section. Here you should see the query that you had saved before, select that.</p> 714<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-6.png?w=1700" alt="Kibana dashboard step 1" /></p> 715<p>To save this dashboard, click on the <code>Save</code> button from the top bar, give your dashboard a name and proceed to click on <code>Save</code>.</p> 716<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-7.png?w=1700" alt="Kibana dashboard step 2" /></p> 717<p>There you have it folks, users can now come to Kibana, look at logs, you can also setup other graphs like errors over time, user hits according to time of day or whatever you can think of.</p> 718<h3 id="short-note-on-s3-storage-mechanisms-archival-options">Short note on S3 storage mechanisms, archival options</h3> 719<p>S3 is a highly reliable redundant object storage service from AWS, along with simply providing space to store files, you can also:</p> 720<ol> 721<li>Manage security/access</li> 722<li>Setup events which can invoke Lambda functions to take some action (move data, setup alerts, send stuff into other databases/ queues for further processing etc)</li> 723<li>Manage archival of data by either deletion or moving to cheaper storage options like <a href="https://aws.amazon.com/glacier/">AWS Glacier</a></li> 724</ol> 725<p>If you want me to write on any of these topics specifically, let me know by commenting below.</p> 726<h2 id="whats-left-to-get-to-the-point-where-i-will-no-longer-need-24x7-support">Whatâs left to get to the point where I will no longer need 24×7 support?</h2> 727<p>Now that we know <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/">what to log</a>, <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">how to get elastic search and kibana up</a> to view the logs and how to get data properly into elastic search <a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/">using AWS Lambda</a> to utilize S3 as blob storage, we need to now learn how to have:</p> 728<ol> 729<li>SNS integration with Pagerduty / emails for real-time alerts which<br /> 730rings your phone for severity 1 issues</li> 731<li>Backup monitoring to monitor Elastic search to build a fool proof auto-alert system in case the primary alert mechanism begins to fail</li> 732</ol> 733<p><strong>Follow this blog/subscribe to email alerts (top of sidebar or below comments in mobile) so that you know
733when any of these posts come about.</strong></p> 734<div id="atatags-1138136615-6ab58dd567449"> 735
735<script type="text/javascript"> 736 __ATA = window.__ATA || {}; 737 __ATA.cmd = window.__ATA.cmd || []; 738 __ATA.cmd.push(function() { 739 __ATA.initVideoSlot('atatags-1138136615-6ab58dd567449', { 740 sectionId: '1138136615', 741 format: 'inread' 742 }); 743 }); 744 </script>
744 745 </div><span id="wordads-inline-marker" style="display: none;"></span><div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-linkedin"><a rel="nofollow noopener noreferrer" 746 data-shared="sharing-linkedin-289" 747 class="share-linkedin sd-button share-icon" 748 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=linkedin" 749 target="_blank" 750 aria-labelledby="sharing-linkedin-289" 751 > 752 <span id="sharing-linkedin-289" hidden>Share on LinkedIn (Opens in new window)</span> 753 <span>LinkedIn</span> 754 </a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer" 755 data-shared="sharing-reddit-289" 756 class="share-reddit sd-button share-icon" 757 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=reddit" 758 target="_blank" 759 aria-labelledby="sharing-reddit-289" 760 > 761 <span id="sharing-reddit-289" hidden>Share on Reddit (Opens in new window)</span> 762 <span>Reddit</span> 763 </a></li><li class="share-tumblr"><a rel="nofollow noopener noreferrer" 764 data-shared="sharing-tumblr-289" 765 class="share-tumblr sd-button share-icon" 766 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=tumblr" 767 target="_blank" 768 aria-labelledby="sharing-tumblr-289" 769 > 770 <span id="sharing-tumblr-289" hidden>Share on Tumblr (Opens in new window)</span> 771 <span>Tumblr</span> 772 </a></li><li class="share-twitter"><a rel="nofollow noopener noreferrer" 773 data-shared="sharing-twitter-289" 774 class="share-twitter sd-button share-icon" 775 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=twitter" 776 target="_blank" 777 aria-labelledby="sharing-twitter-289" 778 > 779 <span id="sharing-twitter-289" hidden>Share on X (Opens in new window)</span> 780 <span>X</span> 781 </a></li><li class="share-facebook"><a rel="nofollow noopener noreferrer" 782 data-shared="sharing-facebook-289" 783 class="share-facebook sd-button share-icon" 784 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=facebook" 785 target="_blank" 786 aria-labelledby="sharing-facebook-289" 787 > 788 <span id="sharing-facebook-289" hidden>Share on Facebook (Opens in new window)</span> 789 <span>Facebook</span> 790 </a></li><li class="share-pinterest"><a rel="nofollow noopener noreferrer" 791 data-shared="sharing-pinterest-289" 792 class="share-pinterest sd-button share-icon" 793 href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=pinterest" 794 target="_blank" 795 aria-labelledby="sharing-pinterest-289" 796 > 797 <span id="sharing-pinterest-289" hidden>Share on Pinterest (Opens in new window)</span> 798 <span>Pinterest</span> 799 </a></li><li class="share-email"><a rel="nofollow noopener noreferrer" 800 data-shared="sharing-email-289" 801 class="share-email sd-button share-icon" 802 href="mailto:?subject=%5BShared%20Post%5D%20Scaling%20for%20TB%20level%20log%20data%20with%20Kinesis%20Firehose%20and%20lambda%20with%20Elastic%20Search%20on%20AWS&body=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&share=email" 803 target="_blank" 804 aria-labelledby="sharing-email-289" 805 data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you're having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="8446e5c825" data-email-sh
805are-track-url="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=email"> 806 <span id="sharing-email-289" hidden>Email a link to a friend (Opens in new window)</span> 807 <span>Email</span> 808 </a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-113813661-289-6ab58dd568836' data-src='//widgets.wp.com/likes/index.html?ver=20260924#blog_id=113813661&post_id=289&origin=cloudncode.wordpress.com&obj_id=113813661-289-6ab58dd568836&domain=cloudncode.blog' data-name='like-post-frame-113813661-289-6ab58dd568836' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div></div> </div><!-- .entry-content --> 809 810 <footer class="entry-meta"> 811 <a class="entry-date" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/">February 6, 2017</a> in <a href="https://cloudncode.blog/category/cloud/" rel="category tag">Cloud</a>, <a href="https://cloudncode.blog/category/code/" rel="category tag">Code</a>, <a href="https://cloudncode.blog/category/serverless-design/" rel="category tag">Serverless Design</a>. Tags: <a href="https://cloudncode.blog/tag/elasticsearch/" rel="tag">elasticsearch</a>, <a href="https://cloudncode.blog/tag/elasticsearch-on-aws/" rel="tag">elasticsearch on aws</a>, <a href="https://cloudncode.blog/tag/getting-kinesis-firehose-to-put-data-in-elastic-search/" rel="tag">Getting kinesis firehose to put data in elastic search</a>, <a href="https://cloudncode.blog/tag/getting-started-with-kinesis-in-aws/" rel="tag">getting started with Kinesis in AWS</a>, <a href="https://cloudncode.blog/tag/how-to-handle-terrabyte-level-log-data-in-aws/" rel="tag">how to handle terrabyte level log data in aws</a>, <a href="https://cloudncode.blog/tag/how-to-horizontally-scale-log-storage-on-aws/" rel="tag">how to horizontally scale log storage on aws</a>, <a href="https://cloudncode.blog/tag/kinesis-firehose/" rel="tag">Kinesis firehose</a>, <a href="https://cloudncode.blog/tag/using-aws-lambda-with-kinesis/" rel="tag">using AWS Lambda with Kinesis</a> 812 </footer><!-- .entry-meta --> 813</article><!-- #post-## --> 814 <div class="related-content"> 815 <h3 class="related-content-title">Related posts</h3> 816 817 <article id="post-181" class="post-181 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud tag-aws tag-aws-elastic-search-service-getting-started tag-cloud-computing tag-elastic-search-5-getting-started-aws tag-elastic-search-and-kibana tag-elastic-search-on-cloud-getting-started"> 818 819 <div class="entry-thumbnail"> 820 <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=50&h=50&crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=50&h=50&crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=100&h=100&crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="286" data-permalink="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/es-5-1-step-8/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png" data-orig-size="1909,934" data-comments-opened="1" data-image-title="es-5-1-step-8" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=700" /></a> 821 </div> 822 823 <header class="entry-header"> 824 <h3 class="entry-title"><a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/" rel="bookmark">How to Set up Elastic Search & Kibana on AWS</a></h3> 825 </header><!-- .entry-header --> 826 827 </article> 828 829 830 <article id="post-340" class="post-340 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud category-code category-serverless-design tag-aws-lambda tag-aws-lambda-concurrency tag-aws-lambda-function-best-practices tag-aws-lambda-logging-best-practices tag-aws-lambda-performance-considerations tag-aws-lambda-stateless-design tag-choosing-the-right-aws-lambda-function-size tag-how-does-aws-lambda-scale tag-serverless-with-aws tag-streaming-aws-cloudwatch-logs-into-elasticsearch"> 831
832 <div class="entry-thumbnail"> 833 <a href="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=50&h=50&crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=50&h=50&crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=100&h=100&crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="372" data-permalink="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/2000px-orange_lambda-svg/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png" data-orig-size="2000,2090" data-comments-opened="1" data-image-title="2000px-orange_lambda-svg" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=700" /></a> 834 </div> 835 836 <header class="entry-header"> 837 <h3 class="entry-title"><a href="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/" rel="bookmark">Best practices – AWS Lambda function</a></h3> 838 </header><!-- .entry-header --> 839 840 </article> 841 842 843 <article id="post-219" class="post-219 post type-post status-publish format-standard has-post-thumbnail hentry category-code category-serverless-design tag-net-core tag-aws-kinesis tag-aws-lambda tag-aws-lambda-example-c tag-c tag-getting-started-with-visual-studio-code tag-getting-started-with-visual-studio-code-and-net-core tag-visual-studio-code"> 844 845 <div class="entry-thumbnail"> 846 <a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=50&h=50&crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=50&h=50&crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=100&h=100&crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="226" data-permalink="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/dot-net-lambda-help/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png" data-orig-size="909,499" data-comments-opened="1" data-image-title="dot-net-lambda-help" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=700" /></a> 847 </div> 848 849 <header class="entry-header"> 850 <h3 class="entry-title"><a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/" rel="bookmark">Getting started with writing and debugging AWS Lambda Function with Visual Studio Code</a></h3> 851 </header><!-- .entry-header --> 852 853 </article> 854 855 </div> 856 857 <nav role="navigation" id="nav-below" class="navigation-post"> 858 <h1 class="screen-reader-text">Post navigation</h1> 859 860 861 <div class="nav-previous"><a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/" rel="prev"><span class="meta-nav">←</span> Getting started with writing and debugging AWS Lambda Function with Visual Studio Code</a></div> <div class="nav-next"><a href="https://cloudncode.blog/2017/02/13/net-core-web-api-lambda-performance/" rel="next">.Net Core Web API on AWS Lambda Performance <span class="meta-nav">→</span></a></div> 862 863 </nav><!-- #nav-below --> 864 865 866 <div id="comments" class="comments-area"> 867 868 869 <h2 class="comments-title"> 870 4 thoughts on “<span>Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS</span>” </h2> 871 872 873 <ol class="comment-list"> 874 <li class="post pingback"> 875 <p>Pingback: <a href="http://zerozone.com/mmp/aceinfoway/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws-/" class="url" rel="ugc external nofollow">Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS | Ace Infoway</a></p> 876 </li><!-- #comment-## --> 877 <li class="post pingback"> 878 <p>Pingback: <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/" class="url" rel="ugc">What should you log in an application and how to avoid having 24×
8787 support looking at them? | cloudncode</a></p> 879 </li><!-- #comment-## --> 880 <li class="comment even thread-even depth-1" id="li-comment-213"> 881 <article id="comment-213" class="comment"> 882 <footer> 883 <div class="comment-author vcard"> 884 <img referrerpolicy="no-referrer" alt='Prune's avatar' src='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&d=identicon&r=R' srcset='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&d=identicon&r=R 1x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=60&d=identicon&r=R 1.5x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=80&d=identicon&r=R 2x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=120&d=identicon&r=R 3x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=160&d=identicon&r=R 4x' class='avatar avatar-40' height='40' width='40' loading='lazy' decoding='async' /> <cite class="fn">Prune</cite> <span class="says">says:</span> </div><!-- .comment-author .vcard --> 885 886 <div class="comment-meta commentmetadata"> 887 <a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comment-213"><time datetime="2017-03-02T21:16:09+05:30"> 888 March 2, 2017 at 9:16 pm </time></a> 889 </div><!-- .comment-meta .commentmetadata --> 890 </footer> 891 892 <div class="comment-content"><p>I don’t understand. You’re saying “you don’t need everything or unprocessed logs inside ES”, but as far as I understant, Kinesis is just your “broker” and will ship everything into ES… right ?</p> 893<p id="comment-like-213" data-liked=comment-not-liked class="comment-likes comment-not-liked"><a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?like_comment=213&_wpnonce=fd610c62b0" class="comment-like-link needs-login" rel="nofollow" data-blog="113813661"><span>Like</span></a><span id="comment-like-count-213" class="comment-like-feedback">Like</span></p> 894</div> 895 896 <div class="reply"> 897 <a rel="nofollow" class="comment-reply-link" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?replytocom=213#respond" data-commentid="213" data-postid="289" data-belowelement="comment-213" data-respondelement="respond" data-replyto="Reply to Prune" aria-label="Reply to Prune">Reply</a> </div><!-- .reply --> 898 </article><!-- #comment-## --> 899 900 <ul class="children"> 901 <li class="comment byuser comment-author-prune998 odd alt depth-2" id="li-comment-214"> 902 <article id="comment-214" class="comment"> 903 <footer> 904 <div class="comment-author vcard"> 905 <img referrerpolicy="no-referrer" alt='prune998's avatar' src='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&d=identicon&r=R' srcset='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&d=identicon&r=R 1x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=60&d=identicon&r=R 1.5x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=80&d=identicon&r=R 2x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=120&d=identicon&r=R 3x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=160&d=identicon&r=R 4x' class='avatar avatar-40' height='40' width='40' loading='lazy' decoding='async' /> <cite class="fn">prune998</cite> <span class="says">says:</span> </div><!-- .comment-author .vcard --> 906 907 <div class="comment-meta commentmetadata"> 908 <a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comment-214"><time datetime="2017-03-02T21:17:12+05:30"> 909 March 2, 2017 at 9:17 pm </time></a> 910 </div><!-- .comment-meta .commentmetadata --> 911 </footer> 912 913 <div class="comment-content"><p>Sorry, in fact the Lambda function could be extended to filter some logs…</p>
914<p id="comment-like-214" data-liked=comment-not-liked class="comment-likes comment-not-liked"><a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?like_comment=214&_wpnonce=e59dcbc8f6" class="comment-like-link needs-login" rel="nofollow" data-blog="113813661"><span>Like</span></a><span id="comment-like-count-214" class="comment-like-feedback">Liked by <a href="#" class="view-likers" data-like-count="1">1 person</a></span></p> 915</div> 916 917 <div class="reply"> 918 <a rel="nofollow" class="comment-reply-link" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?replytocom=214#respond" data-commentid="214" data-postid="289" data-belowelement="comment-214" data-respondelement="respond" data-replyto="Reply to prune998" aria-label="Reply to prune998">Reply</a> </div><!-- .reply --> 919 </article><!-- #comment-## --> 920 921 </li><!-- #comment-## --> 922</ul><!-- .children --> 923</li><!-- #comment-## --> 924 </ol><!-- .comment-list --> 925 926 927 928 929 <div id="respond" class="comment-respond"> 930 <h3 id="reply-title" class="comment-reply-title">Leave a comment <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://cloudncode.blog/wp-comments-post.php" method="post" id="commentform" class="comment-form"> 931 932 933<div class="comment-form__verbum transparent"></div><div class="verbum-form-meta"><input type='hidden' name='comment_post_ID' value='289' id='comment_post_ID' /> 934<input type='hidden' name='comment_parent' id='comment_parent' value='0' /> 935 936 <input type="hidden" name="highlander_comment_nonce" id="highlander_comment_nonce" value="3dff5f3ccb" /> 937 <input type="hidden" name="verbum_show_subscription_modal" value="" /></div><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="b959427df8" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>Δ<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="94"/>
937<script> 938document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() ); 939</script>
939 940</p></form> </div><!-- #respond --> 941 942</div><!-- #comments --> 943 944 945 </div><!-- #content --> 946 </div><!-- #primary --> 947 948 <div id="secondary" class="widget-area" role="complementary"> 949 <aside id="rss-3" class="widget widget_rss"><h1 class="widget-title"><a class="rsswidget" href="https://cloudncode.blog/feed/" title="Syndicate this content"><img style="background: orange; color: white; border: none;" width="14" height="14" src="https://s.wordpress.com/wp-includes/images/rss.png?m=1354137473i" alt="RSS" /></a> <a class="rsswidget" href="https://cloudncode.blog" title="Cloud Computing and the Code that Powers it">RSS</a></h1><ul><li><a class='rsswidget' href='https://cloudncode.blog/2018/03/27/tidbit-getting-started-with-aws-lambda-function-using-nodejs/' title='This is a Tidbit, basically whenever you see Tidbit: in the title, it means I am going to simply throw some helpful code without explaining too much around it. What will you be able to do after going through this? Build & deploy a REST API on AWS Lambda using nodeJS which uses AWS APIâ¦'>Tidbit: Getting started with AWS Lambda function using nodeJS</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2018/03/22/is-aws-lambda-expensive-to-run/' title='What the pricing pivots on, explain the pricing model for AWS Lambda, what you should look out for when it comes to pricing. After reading this article, you will be able to make a more educated decision on whether AWS Lambda (or serverless architecture in general) is right for you or not.'>Is AWS Lambda expensive to run?</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/11/07/make-your-first-iot-device-via-aws-iot-service-and-raspberry-pi/' title='IOT or Internet of Things is a hot topic these days, but more than that its exciting. I have been a developer for more than 10 years now and although I have designed and built applications that can handle millions of calls and analyze terabytes of data in real time, the most exciting thing Iâ¦'>Make your first IOT device via AWS IOT Service and Raspberry Pi</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/08/08/tidbit-api-gateway-as-a-s3-proxy-cloudformation-script-with-serverless-framework/' title='Quick helpful article to get you to setup an API gateway which acts as a S3 proxy using Cloudformation script. This is a Tidbit, basically whenever you see Tidbit: in the title, it means I am going to simply throw some helpful code without explaining too much around it. What you can use this for?â¦'>Tidbit: API Gateway as a S3 Proxy â CloudFormation script with Serverless Framework</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/08/01/serverless-ci-part-2-getting-started-with-aws-codepipeline/' title='This is the second and final part of Serverless CI. In the previous post I walked us through setting up an AWS CodeBuild job which allocates a server, runs your build job and then deallocates it on demand. In this post we are going to walkthrough creating an AWS CodePipeline job which would automate theseâ¦'>Serverless CI: Part 2 â Getting started with AWS CodePipeline</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/06/13/serverless-ci-part-1-getting-started-with-aws-codebuild/' title='On my road to Serverless and my mission to spend less time on maintenance and more on product development, I finally decided to let go of Jenkins and go for a Serverless version of CI. As I have everything else in AWS, AWS CodeBuild and CodePipeline seemed to be the perfect fit for my purpose.â¦'>Serverless CI: Part 1 â Getting started with AWS CodeBuild</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/05/15/why-serverless-architecture/' title='At Tavisca Solutions we started developing beats a few months ago and as the product discovery went to a stage when we wanted to begin with a fast iterative product which can quickly evolve to MVP (minimum viable product) stage we were faced with a rare opportunity. Every time we start a new product weâ¦'>Why Serverless architecture?</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/04/14/getting-started-with-the-serverless-framework-on-aws/' title='I have been in hibernation for some time as I shifted jobs, moved houses and have been hellishly busy, so, my bad that I havenât written an article in a while. In this article, I am going to introduce and help you get started with the serverless tool where we are going to create aâ¦'>Getting Started with the Serverless framework on AWS</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/03/05/automation-tool-for-aws-lambda-sizing/' title='As a continuation to my post on best practices on AWS Lambda I wrote a tool which finds the right size of your existing Lambda function programmatically by continuously resizing it, firing test events at it until it finds the best size which would either: Save you the most money OR Give you the bestâ¦'>Automation tool for AWS Lambda sizing</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/' title='In this post we are going to go through the best practices while building an AWS Lambda function. In order to go through this blog you should know what is it and
949you should ideally have built at least a simple function around it so you have your bearings right, you can checkout this blogâ¦'>Best practices â AWS Lambda function</a></li></ul></aside><aside id="blog_subscription-5" class="widget widget_blog_subscription jetpack_subscription_widget"><h1 class="widget-title"><label for="subscribe-field">Follow Blog via Email</label></h1> 950 951 <div class="wp-block-jetpack-subscriptions__container"> 952 <form 953 action="https://subscribe.wordpress.com" 954 method="post" 955 accept-charset="utf-8" 956 data-blog="113813661" 957 data-post_access_level="everybody" 958 id="subscribe-blog" 959 > 960 <p>Enter your email address to follow this blog and receive notifications of new posts by email.</p> 961 <p id="subscribe-email"> 962 <label 963 id="subscribe-field-label" 964 for="subscribe-field" 965 class="screen-reader-text" 966 > 967 Email Address: </label> 968 969 <input 970 type="email" 971 name="email" 972 autocomplete="email" 973 974 style="width: 95%; padding: 1px 10px" 975 placeholder="Email Address" 976 value="" 977 id="subscribe-field" 978 required 979 /> </p> 980 981 <p id="subscribe-submit" 982 > 983 <input type="hidden" name="action" value="subscribe"/> 984 <input type="hidden" name="blog_id" value="113813661"/> 985 <input type="hidden" name="source" value="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/"/> 986 <input type="hidden" name="sub-type" value="widget"/> 987 <input type="hidden" name="redirect_fragment" value="subscribe-blog"/> 988 <input type="hidden" id="_wpnonce" name="_wpnonce" value="d0d76a4c49" /> <button type="submit" 989 class="wp-block-button__link" 990 > 991 Follow </button> 992 </p> 993 </form> 994 </div> 995 996</aside><aside id="twitter_timeline-3" class="widget widget_twitter_timeline"><a class="twitter-timeline" data-height="400" data-theme="light" data-border-color="#e8e8e8" data-lang="EN" data-partner="jetpack" data-chrome="nofooter noborders transparent" href="https://twitter.com/maingified" href="https://twitter.com/maingified">My Tweets</a></aside> </div><!-- #secondary --> 997 </div><!-- #main --> 998 999 <footer id="colophon" class="site-footer" role="contentinfo"> 1000 <div class="site-info"> 1001 <a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a> 1002 </div><!-- .site-info --> 1003 </footer><!-- #colophon --> 1004</div><!-- #page --> 1005 1006<!-- -->
1007<script type="speculationrules"> 1008{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/expound/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]} 1009</script>
vendor: 1 bytes, line 1009
1009
1010<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202639d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
vendor: 1 bytes, line 1010
1010
1011<script id="wpgroho-js-extra"> 1012var WPGroHo = {"my_hash":""}; 1013//# sourceURL=wpgroho-js-extra 1014</script>
vendor: 1 bytes, line 1014
1014
1015<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
1015 1016 1017
1017<script> 1018 // Initialize and attach hovercards to all gravatars 1019 ( function() { 1020 function init() { 1021 if ( typeof Gravatar === 'undefined' ) { 1022 return; 1023 } 1024 1025 if ( typeof Gravatar.init !== 'function' ) { 1026 return; 1027 } 1028 1029 Gravatar.profile_cb = function ( hash, id ) { 1030 WPGroHo.syncProfileData( hash, id ); 1031 }; 1032 1033 Gravatar.my_hash = WPGroHo.my_hash; 1034 Gravatar.init( 1035 'body', 1036 '#wp-admin-bar-my-account', 1037 { 1038 i18n: { 1039 'Edit your profile â': 'Edit your profile â', 1040 'View profile â': 'View profile â', 1041 'Contact': 'Contact', 1042 'Send money': 'Send money', 1043 'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.', 1044 'Gravatar not found.': 'Gravatar not found.', 1045 'Too Many Requests.': 'Too Many Requests.', 1046 'Internal Server Error.': 'Internal Server Error.', 1047 'Is this you?': 'Is this you?', 1048 'Claim your free profile.': 'Claim your free profile.', 1049 'Email': 'Email', 1050 'Home Phone': 'Home Phone', 1051 'Work Phone': 'Work Phone', 1052 'Cell Phone': 'Cell Phone', 1053 'Contact Form': 'Contact Form', 1054 'Calendar': 'Calendar', 1055 }, 1056 } 1057 ); 1058 } 1059 1060 if ( document.readyState !== 'loading' ) { 1061 init(); 1062 } else { 1063 document.addEventListener( 'DOMContentLoaded', init ); 1064 } 1065 } )(); 1066 </script>
1066 1067 1068 <div style="display:none"> 1069 <div class="grofile-hash-map-92c17cd84fa39613655d7b9be4e612f0"> 1070 </div> 1071 <div class="grofile-hash-map-92c17cd84fa39613655d7b9be4e612f0"> 1072 </div> 1073 </div> 1074 <!-- CCPA [start] --> 1075
1075<script type="text/javascript"> 1076 ( function () { 1077 1078 var setupPrivacy = function() { 1079 1080 // Minimal Mozilla Cookie library 1081 // https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie/Simple_document.cookie_framework 1082 var cookieLib = window.cookieLib = {getItem:function(e){return e&&decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*"+encodeURIComponent(e).replace(/[\-\.\+\*]/g,"\\$&")+"\\s*\\=\\s*([^;]*).*$)|^.*$"),"$1"))||null},setItem:function(e,o,n,t,r,i){if(!e||/^(?:expires|max\-age|path|domain|secure)$/i.test(e))return!1;var c="";if(n)switch(n.constructor){case Number:c=n===1/0?"; expires=Fri, 31 Dec 9999 23:59:59 GMT":"; max-age="+n;break;case String:c="; expires="+n;break;case Date:c="; expires="+n.toUTCString()}return"rootDomain"!==r&&".rootDomain"!==r||(r=(".rootDomain"===r?".":"")+document.location.hostname.split(".").slice(-2).join(".")),document.cookie=encodeURIComponent(e)+"="+encodeURIComponent(o)+c+(r?"; domain="+r:"")+(t?"; path="+t:"")+(i?"; secure":""),!0}}; 1083 1084 // Implement IAB USP API. 1085 window.__uspapi = function( command, version, callback ) { 1086 1087 // Validate callback. 1088 if ( typeof callback !== 'function' ) { 1089 return; 1090 } 1091 1092 // Validate the given command. 1093 if ( command !== 'getUSPData' || version !== 1 ) { 1094 callback( null, false ); 1095 return; 1096 } 1097 1098 // Check for GPC. If set, override any stored cookie. 1099 if ( navigator.globalPrivacyControl ) { 1100 callback( { version: 1, uspString: '1YYN' }, true ); 1101 return; 1102 } 1103
1104 // Check for cookie. 1105 var consent = cookieLib.getItem( 'usprivacy' ); 1106 1107 // Invalid cookie. 1108 if ( null === consent ) { 1109 callback( null, false ); 1110 return; 1111 } 1112 1113 // Everything checks out. Fire the provided callback with the consent data. 1114 callback( { version: 1, uspString: consent }, true ); 1115 }; 1116 1117 // Initialization. 1118 document.addEventListener( 'DOMContentLoaded', function() { 1119 1120 // Internal functions. 1121 var setDefaultOptInCookie = function() { 1122 var value = '1YNN'; 1123 var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname; 1124 cookieLib.setItem( 'usprivacy', value, 365 * 24 * 60 * 60, '/', domain ); 1125 }; 1126 1127 var setDefaultOptOutCookie = function() { 1128 var value = '1YYN'; 1129 var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname; 1130 cookieLib.setItem( 'usprivacy', value, 24 * 60 * 60, '/', domain ); 1131 }; 1132 1133 var setDefaultNotApplicableCookie = function() { 1134 var value = '1---'; 1135 var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname; 1136 cookieLib.setItem( 'usprivacy', value, 24 * 60 * 60, '/', domain ); 1137 }; 1138 1139 var setCcpaAppliesCookie = function( applies ) { 1140 var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname; 1141 cookieLib.setItem( 'ccpa_applies', applies, 24 * 60 * 60, '/', domain ); 1142 } 1143 1144 var maybeCallDoNotSellCallback = function() { 1145 if ( 'function' === typeof window.doNotSellCallback ) { 1146 return window.doNotSellCallback(); 1147 } 1148 1149 return false; 1150 } 1151 1152 // Look for usprivacy cookie first. 1153 var usprivacyCookie = cookieLib.getItem( 'usprivacy' ); 1154 1155 // Found a usprivacy cookie. 1156 if ( null !== usprivacyCookie ) { 1157 1158 // If the cookie indicates that CCPA does not apply, then bail. 1159 if ( '1---' === usprivacyCookie ) { 1160 return; 1161 } 1162 1163 // CCPA applies, so call our callback to add Do Not Sell link to the page. 1164 maybeCallDoNotSellCallback(); 1165 1166 // We're all done, no more processing needed. 1167 return; 1168 } 1169 1170 // We don't have a usprivacy cookie, so check to see if we have a CCPA applies cookie. 1171 var ccpaCookie = cookieLib.getItem( 'ccpa_applies' ); 1172 1173 // No CCPA applies cookie found, so we'll need to geolocate if this visitor is from California. 1174 // This needs to happen client side because we do not have region geo data in our $SERVER headers, 1175 // only country data -- therefore we can't vary cache on the region. 1176 if ( null === ccpaCookie ) { 1177 1178 var request = new XMLHttpRequest(); 1179 request.open( 'GET', 'https://public-api.wordpress.com/geo/', true ); 1180 1181 request.onreadystatechange = function () { 1182 if ( 4 === this.readyState ) { 1183 if ( 200 === this.status ) { 1184 1185 // Got a geo response. Parse out the region data. 1186 var data = JSON.parse( this.response ); 1187 var region = data.region ? data.region.toLowerCase() : ''; 1188 var ccpa_applies = ['california', 'colorado', 'connecticut', 'delaware', 'indiana', 'iowa', 'montana', 'new jersey', 'oregon', 'tennessee', 'texas', 'utah', 'virginia'].indexOf( region ) > -1; 1189 // Set CCPA applies cookie. This keeps us from having to make a geo request too frequently. 1190 setCcpaAppliesCookie( ccpa_applies ); 1191 1192 // Check if CCPA applies to set the proper usprivacy cookie. 1193 if ( ccpa_applies ) { 1194 if ( maybeCallDoNotSellCallback() ) { 1195 // Do Not Sell link added, so set default opt-in. 1196 setDefaultOptInCookie(); 1197 } else { 1198 // Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe. 1199 setDefaultOptOutCookie(); 1200 } 1201 } else { 1202 // CCPA does not apply. 1203 setDefaultNotApplicableCookie(); 1204 } 1205 } else { 1206 // Could not geo, so let's assume for now that CCPA applies to be safe. 1207 setCcpaAppliesCookie( true ); 1208 if ( maybeCallDoNotSellCallback() ) { 1209 // Do Not Sell link added, so set default opt-in. 1210 setDefaultOptInCookie(); 1211 } else { 1212 // Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe. 1213 setDefaultOptOutCookie(); 1214 } 1215 } 1216 } 1217 }; 1218 1219 // Send the geo request. 1220 request.send(); 1221 } else { 1222 // We found a CCPA applies cookie. 1223 if ( ccpaCookie === 'true' ) { 1224 if ( maybeCallDoNotSellCallback() ) { 1225 // Do Not Sell link added, so set default opt-in. 1226 setDefaultOptInCookie(); 1227 } else { 1228 // Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe. 1229 setDefaultOptOutCookie(); 1230 } 1231 } else { 1232 // CCPA does not apply. 1233 setDefaultNotApplicableCookie(); 1234 } 1235 } 1236 } ); 1237 }; 1238 1239 // Kickoff initialization. 1240 if ( window.defQueue && defQueue.isLOHP && defQueue.isLOHP === 2020 ) { 1241 defQueue.items.push( setupPrivacy ); 1242 } else { 1243 setupPrivacy(); 1244 } 1245 1246 } )(); 1247 </script>
1247 1248 1249 <!-- CCPA [end] --> 1250 <div class="widget widget_eu_cookie_law_widget"> 1251<div 1252 class="hide-on-button ads-active" 1253 data-hide-timeout="30" 1254 data-consent-expiration="180" 1255 id="eu-cookie-law" 1256 style="display: none" 1257> 1258 <form method="post"> 1259 <input type="submit" value="Close and accept" class="accept" /> 1260 1261 Privacy & Cookies: This site uses cookies. By continuing to use this website, you agree to their use. <br /> 1262To find out more, including how to control cookies, see here: 1263 <a href="https://automattic.com/cookies/" rel="nofollow"> 1264 Cookie Policy </a> 1265 </form> 1266</div> 1267</div> <div id="actionbar" dir="ltr" style="display: none;" 1268 class="actnbr-pub-expound" role="region" aria-label="Site actions"> 1269 <span class="actnbr-live" aria-live="polite"></span> 1270 <ul> 1271 <li class="actnbr-btn"> 1272 <a class="actnbr-action actnbr-actn-comment" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comments"> 1273 <svg class="actnbr-icon actnbr-icon-comment" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M18 4H6c-1.1 0-2 .9-2 2v12.9c0 .6.5 1.1 1.1 1.1.3 0 .5-.1.8-.3L8.5 17H18c1.1 0 2-.9 2-2V6c0-1.1-.9-2-2-2zm.5 11c0 .3-.2.5-.5.5H7.9l-2.4 2.4V6c0-.3.2-.5.5-.5h12c.3 0 .5.2.5.5v9z"/></svg> <span>Comment </span> 1274 </a> 1275 </li> 1276 <li class="actnbr-btn"> 1277 <a class="actnbr-action actnbr-actn-reblog" href="" role="button"> 1278 <svg class="actnbr-icon actnbr-icon-reusable-block" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M7 7.2h8.2L13.5 9l1.1 1.1 3.6-3.6-3.5-4-1.1 1 1.9 2.3H7c-.9 0-1.7.3-2.3.9-1.4 1.5-1.4 4.2-1.4 5.6v.2h1.5v-.3c0-1.1 0-3.5 1-4.5.3-.3.7-.5 1.2-.5zm13.8 4V11h-1.5v.3c0 1.1 0 3.5-1 4.5-.3.3-.7.5-1.3.5H8.8l1.7-1.7-1.1-1.1L5.9 17l3.5 4 1.1-1-1.9-2.3H17c.9 0 1.7-.3 2.3-.9 1.5-1.4 1.5-4.2 1.5-5.6z"/></svg><span>Reblog</span> 1279 </a> 1280 </li> 1281 <li class="actnbr-btn actnbr-hidden"> 1282 <a class="actnbr-action actnbr-actn-follow " href="" role="button" aria-haspopup="dialog" aria-expanded="false"> 1283 <svg class="actnbr-icon actnbr-icon-bell" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M17 11.5c0 1.353.17 2.368.976 3 .266.209.602.376 1.024.5v1H5v-1c.422-.124.757-.291 1.024-.5.806-.632.976-1.647.976-3V9c0-2.8 2.2-5 5-5s5 2.2 5 5v2.5ZM15.5 9v2.5c0 .93.066 1.98.515 2.897l.053.103H7.932a4.018 4.018 0 0 0 .053-.103c.449-.917.515-1.967.515-2.897V9c0-1.972 1.528-3.5 3.5-3.5s3.5 1.528 3.5 3.5Zm-5.492 9.008c0-.176.023-.346.065-.508h3.854A1.996 1.996 0 0 1 12 20c-1.1 0-1.992-.892-1.992-1.992Z"/></svg> <span>Subscribe</span> 1284 </a> 1285 <a class="actnbr-action actnbr-actn-following no-display" href="" role="button"> 1286 <svg class="actnbr-icon actnbr-icon-check" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" style="fill: none" stroke="currentColor" stroke-width="1.5"><path d="M7 12L10 15L17 8" vector-effect="non-scaling-stroke"/></svg> <span>Subscribed</span> 1287 </a> 1288 <div class="actnbr-popover actnbr-panel actnbr-notice" id="follow-bubble" role="dialog" aria-label="Subscribe to cloudncode"> 1289 <div class="actnbr-follow-bubble"> 1290 <div class="actnbr-panel__group"><a class="actnbr-menu__item actnbr-sitename" href="https://cloudncode.blog"><img loading='lazy' alt='' src='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png?m=1479929237i' srcset='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png 1x' class='avatar avatar-50' height='50' width='50' /><span class="actnbr-menu__label">cloudncode</span></a></div> <div class="actnbr-panel__group"> 1291 <div class="actnbr-message no-display" aria-live="polite"></div> 1292 <form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" class="no-display"> 1293 <div class="actnbr-follow-count">Join 112 other subscribers</div> 1294 <input type="email" name="email" placeholder="Want updates? Enter your email" class="actnbr-email-field" aria-label="Email address" /> 1295 <input type="hidden" name="action" value="subscribe" /> 1296 <input type="hidden" name="blog_id" value="113813661" /> 1297 <input type="hidden" name="source" value="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" /> 1298 <input type="hidden" name="sub-type" value="actionbar-follow" /> 1299 <input type="hidden" id="_wpnonce" name="_wpnonce" value="d0d76a4c49" /> <button type="submit" class="actnbr-button">Sign me up</button> 1300 </form> 1301 <p class="actnbr-login-nudge"> 1302 Have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fcloudncode.blog%252F2017%252F02%252F06%252Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%252F">Log in now.</a> </p> 1303 </div> 1304 </div> 1305 </div> 1306 </li> 1307 <li class="actnbr-ellipsis actnbr-hidden"> 1308 <button type="button" class="actnbr-more-toggle" aria-haspopup="true" aria-expanded="false" aria-label="More options"> 1309 <svg class="actnbr-icon actnbr-icon-more-horizontal" width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M11 13h2v-2h-2v2zm-6 0h2v-2H5v2zm12-2v2h2v-2h-2z"/></svg> </button> 1310 <div class="actnbr-popover actnbr-menu" role="menu" aria-label="Site options"> 1311 <div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-sitename" href="https://cloudncode.blog"><img loading='lazy' alt='' src='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png?m=1479929237i' srcset='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png 1x' class='avatar avatar-50' height='50' width='50' /><span class="actnbr-menu__label">cloudncode</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-shortlink" href="https://wp.me/p7Hy8d-4F"><span class="actnbr-menu__label actnbr-shortlink__text">Copy shortlink</span><span class="actnbr-menu__icon actnbr-shortlink__icon"><svg class="actnbr-icon actnbr-icon-copy" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M5 4.5h11a.5.5 0 0 1 .5.5v11a.5.5 0 0 1-.5.5H5a.5.5 0 0 1-.5-.5V5a.5.5 0 0 1 .5-.5ZM3 5a2 2 0 0 1 2-2h11a2 2 0 0 1 2 2v11a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5Zm17 3v10.75c0 .69-.56 1.25-1.25 1.25H6v1.5h12.75a2.75 2.75 0 0 0 2.75-2.75V8H20Z"/></svg></span><span class="actnbr-menu__icon actnbr-shortlink__icon-copied"><svg class="actnbr-icon actnbr-icon-check" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" style="fill: none" stroke="currentColor" stroke-width="1.5"><path d="M7 12L10 15L17 8" vector-effect="non-scaling-stroke"/></svg></span></a><a role="menuitem" class="actnbr-menu__item actnbr-reader" href="https://wordpress.com/reader/blogs/113813661/posts/289"><span class="actnbr-menu__label">View post in Reader</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-subs" href="https://subscribe.wordpress.com/"><span class="actnbr-menu__label">Manage subscriptions</span></a><a role="menuitem" class="actnbr-menu__item actnbr-signup" href="https://wordpress.com/start/"><span class="actnbr-menu__label">Sign up</span></a><a role="menuitem" class="actnbr-menu__item actnbr-login" href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fcloudncode.blog%252F2017%252F02%252F06%252Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%252F"><span class="actnbr-menu__label">Log in</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item flb-report" href="https://wordpress.com/abuse/?report_url=https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" target="_blank" rel="noopener noreferrer"><span class="actnbr-menu__label">Report this content</span><span class="actnbr-menu__icon"><svg class="actnbr-icon actnbr-icon-external" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M19.5 4.5h-7V6h4.44l-5.97 5.97 1.06 1.06L18 7.06v4.44h1.5v-7Zm-13 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-3H17v3a.5.5 0 0 1-.5.5h-10a.5.5 0 0 1-.5-.5v-10a.5.5 0 0 1 .5-.5h3V5.5h-3Z"/></svg></span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-fold" href=""><span class="actnbr-menu__label">Collapse this bar</span></a></div> </div> 1312 </li> 1313 </ul> 1314 </div> 1315
1315<script> 1316window.addEventListener( "DOMContentLoaded", function () { 1317 var link = document.createElement( "link" ); 1318 link.href = "\/wp-content\/mu-plugins\/jetpack-plugin\/sun\/jetpack_vendor\/automattic\/jetpack-newsletter\/src\/..\/build\/action-bar.css?ver=9e2715140ce0af99d983"; 1319 link.rel = "stylesheet"; 1320 document.head.appendChild( link ); 1321 1322 var script = document.createElement( "script" ); 1323 script.src = "\/wp-content\/mu-plugins\/jetpack-plugin\/sun\/jetpack_vendor\/automattic\/jetpack-newsletter\/src\/..\/build\/action-bar.js?ver=9e2715140ce0af99d983"; 1324 document.body.appendChild( script ); 1325 } ); 1326</script>
1326 1327 1328
1328<script type="text/javascript"> 1329 window.WPCOM_sharing_counts = {"https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/":289}; 1330 </script>
1330 1331
1331<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJzTLy/QTc7PK0nNK9EvyClNz8wr1i+uzCtJrMjITM/IAeKS1CJMEWP94uSizIISoOIM5/yiVL2sYh19yo1yKiotzvAKBvOpaaRzcEZiUQHQRPtcW0NTcwMjAxNLC9MsAGaiYW4='></script>
vendor: 1 bytes, line 1331
1331
1332<script type='text/javascript'> 1333 (function(){ 1334 var corecss = document.createElement('link'); 1335 var themecss = document.createElement('link'); 1336 var corecssurl = "/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shCore.css?ver=3.0.9b"; 1337 if ( corecss.setAttribute ) { 1338 corecss.setAttribute( "rel", "stylesheet" ); 1339 corecss.setAttribute( "type", "text/css" ); 1340 corecss.setAttribute( "href", corecssurl ); 1341 } else { 1342 corecss.rel = "stylesheet"; 1343 corecss.href = corecssurl; 1344 } 1345 document.head.appendChild( corecss ); 1346 var themecssurl = "/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shThemeDefault.css?m=1363304414i&ver=3.0.9b"; 1347 if ( themecss.setAttribute ) { 1348 themecss.setAttribute( "rel", "stylesheet" ); 1349 themecss.setAttribute( "type", "text/css" ); 1350 themecss.setAttribute( "href", themecssurl ); 1351 } else { 1352 themecss.rel = "stylesheet"; 1353 themecss.href = themecssurl; 1354 } 1355 document.head.appendChild( themecss ); 1356 })(); 1357 SyntaxHighlighter.config.strings.expandSource = '+ expand source'; 1358 SyntaxHighlighter.config.strings.help = '?'; 1359 SyntaxHighlighter.config.strings.alert = 'SyntaxHighlighter\n\n'; 1360 SyntaxHighlighter.config.strings.noBrush = 'Can\'t find brush for: '; 1361 SyntaxHighlighter.config.strings.brushNotHtmlScript = 'Brush wasn\'t configured for html-script option: '; 1362 SyntaxHighlighter.defaults['pad-line-numbers'] = false; 1363 SyntaxHighlighter.defaults['toolbar'] = false; 1364 SyntaxHighlighter.all(); 1365 1366 // Infinite scroll support 1367 document.addEventListener( 'is.post-load', function () { 1368 SyntaxHighlighter.highlight(); 1369 } ); 1370</script>
1370 1371 1372
1372<script type="text/javascript"> 1373 (function () { 1374 var wpcom_reblog = { 1375 source: 'toolbar', 1376 1377 toggle_reblog_box_flair: function (obj_id, post_id) { 1378 1379 // Go to site selector. This will redirect to their blog if they only have one. 1380 const postEndpoint = `https://wordpress.com/post`; 1381 1382 // Ideally we would use the permalink here, but fortunately this will be replaced with the 1383 // post permalink in the editor. 1384 const originalURL = `${ document.location.href }?page_id=${ post_id }`; 1385 1386 const url = 1387 postEndpoint + 1388 '?url=' + 1389 encodeURIComponent( originalURL ) + 1390 '&is_post_share=true' + 1391 '&v=5'; 1392 1393 const redirect = function () { 1394 if ( 1395 ! window.open( url, '_blank' ) 1396 ) { 1397 location.href = url; 1398 } 1399 }; 1400 1401 if ( /Firefox/.test( navigator.userAgent ) ) { 1402 setTimeout( redirect, 0 ); 1403 } else { 1404 redirect(); 1405 } 1406 }, 1407 }; 1408 1409 window.wpcom_reblog = wpcom_reblog; 1410 })(); 1411 </script>
vendor: 1 bytes, line 1411
1411
1412<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJx9jEsOwjAMBS9EcIuqdoU4i2msNKnzoXbI9ckO1AXbeTMPWjFrTkpJ4cnZmcLV+STQ8mHRilkZRUggdITK5n27BrnATxbrN/LWkQpQ7WvePRnGBkqxMCqd+J8f9nu3X5UqbZgs09HlR7yPyzxM0zyMS/gAPatD8g=='></script>
vendor: 1 bytes, line 1412
1412
1413<script id="wp-hooks-js" src="/wp-content/plugins/gutenberg-core/v24.0.0/build/scripts/hooks/index.min.js?m=1789650619i&ver=f0f188028580e8dc1255"></script>
vendor: 1 bytes, line 1413
1413
1414<script id="wp-i18n-js" src="/wp-content/plugins/gutenberg-core/v24.0.0/build/scripts/i18n/index.min.js?m=1789650619i&ver=1dfe7db3940c23ea9216"></script>
vendor: 1 bytes, line 1414
1414
1415<script id="wp-i18n-js-after"> 1416wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } ); 1417//# sourceURL=wp-i18n-js-after 1418</script>
vendor: 1 bytes, line 1418
1418
1419<script id="verbum-settings-js-before"> 1420window.VerbumComments = {"Log in or provide your name and email to leave a reply.":"Log in or provide your name and email to leave a reply.","Log in or provide your name and email to leave a comment.":"Log in or provide your name and email to leave a comment.","Receive web and mobile notifications for posts on this site.":"Receive web and mobile notifications for posts on this site.","Name":"Name","Email (address never made public)":"Email (address never made public)","Website (optional)":"Website (optional)","Leave a reply. (log in optional)":"Leave a reply. (log in optional)","Leave a comment. (log in optional)":"Leave a comment. (log in optional)","Log in to leave a reply.":"Log in to leave a reply.","Log in to leave a comment.":"Log in to leave a comment.","Logged in via %s":"Logged in via %s","Log out":"Log out","Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.":"Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.","Email":"Email","(Address never made public)":"(Address never made public)","Instantly":"Instantly","Daily":"Daily","Reply":"Reply","Comment":"Comment","WordPress":"WordPress","Weekly":"Weekly","Notify me of new posts":"Notify me of new posts","Email me new posts":"Email me new posts","Email me new comments":"Email me new comments","Cancel":"Cancel","Write a comment...":"Write a comment...","Write a reply...":"Write a reply...","Website":"Website","Optional":"Optional","We'll keep you in the loop!":"We'll keep you in the loop!","Loading your comment...":"Loading your comment...","Discover more from":"Discover more from cloudncode","Subscribe now to keep reading and get access to the full archive.":"Subscribe now to keep reading and get access to the full archive.","Continue reading":"Continue reading","Never miss a beat!":"Never miss a beat!","Interested in getting blog post updates? Simply click the button below to stay in the loop!":"Interested in getting blog post updates? Simply click the button below to stay in the loop!","Enter your email address":"Enter your email address","Subscribe":"Subscribe","Comment sent successfully":"Comment sent successfully","Save my name, email, and website in this browser for the next time I comment.":"Save my name, email, and website in this browser for the next time I comment.","hovercardi18n":{"Edit your profile \u2192":"Edit your profile \u2192","View profile \u2192":"View profile \u2192","Contact":"Contact","Send money":"Send money","Gravatar not found.":"Gravatar not found.","This profile is private.":"This profile is private.","Too Many Requests.":"Too Many Requests.","Internal Server Error.":"Internal Server Error.","Sorry, we are unable to load this Gravatar profile.":"Sorry, we are unable to load this Gravatar profile."},"siteId":113813661,"postId":289,"mustLogIn":false,"requireNameEmail":true,"commentRegistration":false,"connectURL":"https://cloudncode.wordpress.com/public.api/connect/?action=request\u0026domain=cloudncode.blog","logoutURL":"https://cloudncode.wordpress.com/wp-login.php?action=logout\u0026_wpnonce=ba114a174e","homeURL":"https://cloudncode.blog/","subscribeToBlog":true,"subscribeToComment":true,"isJetpackCommentsLoggedIn":false,"jetpackUsername":"","jetpackUserId":0,"jetpackSignature":"","jetpackAvatar":"https://0.gravatar.com/avatar/?s=96\u0026amp;d=identicon\u0026amp;r=R","enableBlocks":true,"enableSubscriptionModal":true,"currentLocale":"en","isJetpackComments":false,"allowedBlocks":["core/paragraph","core/list","core/code","core/list-item","core/quote","core/image","core/embed","core/quote","core/code"],"embedNonce":"cb14ccd6fe","verbumBundleUrl":"/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/features/verbum-comments/dist/index.js","isRTL":false,"vbeCacheBuster":1789740582,"iframeUniqueId":0,"colorScheme":false} 1421//# sourceURL=verbum-settings-js-before 1422</script>
vendor: 1 bytes, line 1422
1422
1423<script data-wp-strategy="defer" defer id="verbum-dynamic-loader-js" src="/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/assets/dynamic-loader.js?m=1772522099i&minify=false&ver=8b5296f9195db92c42e4"></script>
vendor: 1 bytes, line 1423
1423
1424<script id="comment-like-js-extra"> 1425var comment_like_text = {"loading":"Loading...","swipeUrl":"https://s2.wp.com/wp-content/mu-plugins/comment-likes/js/lib/swipe.js?m=1382645497i&ver=20131008"}; 1426//# sourceURL=comment-like-js-extra 1427</script>
vendor: 1 bytes, line 1427
1427
1428<script id="sharing-js-js-extra"> 1429var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"}; 1430//# sourceURL=sharing-js-js-extra 1431</script>
vendor: 1 bytes, line 1431
1431
1432<script id="jetpack-script-data-js-before"> 1433window.JetpackScriptData = {"site":{"host":"wpcom","is_wpcom_platform":true}}; 1434//# sourceURL=jetpack-script-data-js-before 1435</script>
vendor: 1 bytes, line 1435
1435
1436<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJydUNFOwzAM/CHSgJi08YD4lMlL3M5t40Sxs25/T+i2qoCEJh4sWee789l2SsZFVmS1oZg0lo5YrIshVMiMNKDY/gfQ9PJkV0I9Yqi0VA4WzykW9l8ShhN1oBT5Mb4MlKo/D6aNrohp6XwXErux+O9JMqbx0gT65b46o0dN4AZToSlV4W1gpfB9tj8h+5gtFI0BVMktKhBBFXsoNPoFFJcpqfGg8MdiGEgCqnltnu2+pl+ANs9k/0DoVdbZQidSxWyUAtY34T8srqfIETJ68P4yt8Td7Y0f4f1lu9u+bTa1+k8eZsov'></script>
vendor: 1 bytes, line 1436
1436
1437<script id="sharing-js-js-after"> 1438window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {}; 1439window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450"; 1440window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {}; 1441window.WPCOM_sharing_popups[ "tumblr" ] = "menubar=1,resizable=1,width=450,height=450"; 1442window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {}; 1443window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350"; 1444window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {}; 1445window.WPCOM_sharing_popups[ "facebook" ] = "menubar=1,resizable=1,width=600,height=400"; 1446//# sourceURL=sharing-js-js-after 1447</script>
vendor: 1 bytes, line 1447
1447
1448<script id="wp-emoji-settings" type="application/json"> 1449{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63587"}} 1450</script>
vendor: 1 bytes, line 1450
1450
1451<script type="module"> 1452/*! This file is auto-generated */ 1453var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))}); 1454//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js 1455</script>
1455 1456 <iframe src='https://widgets.wp.com/likes/master.html?ver=20260924#ver=20260924' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe> 1457 <div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div> 1458
1458<script src="//stats.wp.com/w.js?69" defer></script>
1458
1458<script type="text/javascript"> 1459_tkq = window._tkq || []; 1460_stq = window._stq || []; 1461_tkq.push(['storeContext', {'blog_id':'113813661','blog_tz':'5','user_lang':'en','blog_lang':'en','user_id':'0'}]); 1462 // Prevent sending pageview tracking from WP-Admin pages. 1463 _stq.push(['view', {'blog':'113813661','v':'wpcom','tz':'5','user_id':'0','post':'289','subd':'cloudncode'}]); 1464 _stq.push(['extra', {'crypt':'UE5XaGUuOTlwaD85flAmcm1mcmZsaDhkV11YdWtpP0NsWnVkPS9sL0ViLndld3BuVT01Uj14Ti1yYTBnX3xuYUQ3Qz1oblZbVHV3MTZoVXFwTTVDK2wuPT1KWzJNV19iQk9bTmF0PUUlUlYzbW1nYyxHTnlKQiUsT0hKb2tYK09aZkN6LzJmYVtxMGpoLT9rJnJRJXg4UUR1VWN0dTBZTmxRSlN6JWx4cEtHLXErWnNjRzVselB1MkMsUEF5MkF5VVZ+Zm9tfmhoZG4wW2hOS05+VGRSUXRDcjhLdlg4cnRtZGhRVm9pOSYxMmw0L2tES0NFNU84JUlaJTc5UWFSMTE0T1lfMmJWLWpOVkpZLGZiWE9jYUEsS3luOWc4a2VtcmtGSnlpNjFDPWdJTFMvWlkxLF9jOFNSMUQvM2YmY0xuW1lDK19GN0Y9Q296RlhQampQeXR5SSxBaTlQaEpXU19nVm81X3pQQ10xL01ZZVk4aldGdC1xR1hZVD1DZGZtL1I1U1dCV3krWHYzakZtLmxKRVp8PS9fR3NyfCVNTGxbQ2kyRD8='}]); 1465_stq.push([ 'clickTrackerInit', '113813661', '289' ]); 1466</script>
1466 1467<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript> 1468<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Asia/Kolkata" data-custom-props="{"logged_in":"0","wptheme":"pub\/expound","wptheme_is_block":"0"}" > 1469
1469<script defer src="/wp-content/js/bilmur.min.js?i=19&m=202639"></script>
1469
1470<script> 1471(function() { 1472 'use strict'; 1473 1474 const fetches = {}; 1475 const promises = {}; 1476 const urls = { 1477 'wp-polyfill': '/wp-includes/js/dist/vendor/wp-polyfill.min.js?m=1786440387i&ver=3.15.0', 1478 'verbum': '/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.js?m=1789668866i&minify=false&ver=1277503d9f869014c379' 1479 }; 1480 const loaders = { 1481 'verbum': () => { 1482 fetchExternalScript('wp-polyfill'); 1483 fetchExternalScript('verbum'); 1484 promises['wp-polyfill'] = promises['wp-polyfill'] || loadWPScript('wp-polyfill'); 1485 promises['verbum'] = promises['verbum'] || promises['wp-polyfill'].then( () => loadWPScript('verbum') ); 1486 return promises['verbum']; 1487 }, 1488 1489 }; 1490 const scriptExtras = { 1491 1492 }; 1493 1494 window.WP_Enqueue_Dynamic_Script = { 1495 loadScript: (handle) => { 1496 if (!loaders[handle]) { 1497 console.error('WP_Enqueue_Dynamic_Script: unregistered script `' + handle + '`.'); 1498 } 1499 return loaders[handle](); 1500 } 1501 }; 1502 1503 function fetchExternalScript(handle) { 1504 if (!urls[handle]) { 1505 return Promise.resolve(); 1506 } 1507 1508 fetches[handle] = fetches[handle] || fetch(urls[handle], { mode: 'no-cors' }); 1509 return fetches[handle]; 1510 } 1511 1512 function runExtraScript(handle, type, index) { 1513 const id = 'wp-enqueue-dynamic-script:' + handle + ':' + type + ':' + (index + 1); 1514 const template = document.getElementById(id); 1515 if (!template) { 1516 return Promise.reject(); 1517 } 1518 1519 const script = document.createElement( 'script' ); 1520 script.innerHTML = template.innerHTML; 1521 document.body.appendChild( script ); 1522 return Promise.resolve(); 1523 } 1524 1525 function loadExternalScript(handle) { 1526 if (!urls[handle]) { 1527 return Promise.resolve(); 1528 } 1529 1530 return fetches[handle].then(() => { 1531 return new Promise((resolve, reject) => { 1532 const script = document.createElement('script'); 1533 script.onload = () => resolve(); 1534 script.onerror = (e) => reject(e); 1535 script.src = urls[handle]; 1536 document.body.appendChild(script); 1537 }); 1538 }); 1539 } 1540 1541 function loadExtra(handle, pos) { 1542 const count = (scriptExtras[handle] && scriptExtras[handle][pos]) || 0; 1543 let promise = Promise.resolve(); 1544 1545 for (let i = 0; i < count; i++) { 1546 promise = promise.then(() => runExtraScript(handle, pos, i)); 1547 } 1548 1549 return promise; 1550 } 1551 1552 function loadWPScript(handle) { 1553 // Core loads scripts in this order. See: https://github.com/WordPress/WordPress/blob/a59eb9d39c4fcba834b70c9e8dfd64feeec10ba6/wp-includes/class-wp-scripts.php#L428. 1554 return loadExtra(handle, 'translations') 1555 .then(() => loadExtra(handle, 'before')) 1556 .then(() => loadExternalScript(handle)) 1557 .then(() => loadExtra(handle, 'after')); 1558 } 1559} )(); 1560</script>
1560 1561 1562</body> 1563</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.