PageSourceSearch

https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-w…hose-and-lambda-with-elastic-search-on-aws/

html cloudncode.blog collected 2026-10-02 07:27:39 UTC 140,870 bytes, 1,563 lines download raw bytes

1<!DOCTYPE html>
2<html lang="en">
3<head>
4<meta charset="UTF-8" />
5<meta name="viewport" content="width=device-width, initial-scale=1" />
6<title>Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS | cloudncode</title>
7<link rel="profile" href="http://gmpg.org/xfn/11" />
8<link rel="pingback" href="https://cloudncode.blog/xmlrpc.php" />
9<!--[if lt IE 9]>
10<script src="https://s2.wp.com/wp-content/themes/pub/expound/js/html5.js?m=1376965621i" type="text/javascript"></script>
10
11<![endif]-->
12
13<meta name='robots' content='max-image-preview:large' />
14<meta name="google-site-verification" content="Pq8KitGCEw84V8ljNgiA-qSr6YraJ2vwWb4KUdK_Y6Y" />
15<meta name="msvalidate.01" content="2E7FC93632C2882390CB6894C1657D64" />
16<meta name="p:domain_verify" content="92b724c934777f301a90d32a724a6a50" />
17
18<!-- Async WordPress.com Remote Login -->
19<script id="wpcom_remote_login_js">
20var wpcom_remote_login_extra_auth = '';
21function wpcom_remote_login_remove_dom_node_id( element_id ) {
22	var dom_node = document.getElementById( element_id );
23	if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
24}
25function wpcom_remote_login_remove_dom_node_classes( class_name ) {
26	var dom_nodes = document.querySelectorAll( '.' + class_name );
27	for ( var i = 0; i < dom_nodes.length; i++ ) {
28		dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
29	}
30}
31function wpcom_remote_login_final_cleanup() {
32	wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
33	wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
34	wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
35	wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
36	wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
37	wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
38}
39
40// Watch for messages back from the remote login
41window.addEventListener( "message", function( e ) {
42	if ( e.origin === "https://r-login.wordpress.com" ) {
43		var data = {};
44		try {
45			data = JSON.parse( e.data );
46		} catch( e ) {
47			wpcom_remote_login_final_cleanup();
48			return;
49		}
50
51		if ( data.msg === 'LOGIN' ) {
52			// Clean up the login check iframe
53			wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
54
55			var id_regex = new RegExp( /^[0-9]+$/ );
56			var token_regex = new RegExp( /^.*|.*|.*$/ );
57			if (
58				token_regex.test( data.token )
59				&& id_regex.test( data.wpcomid )
60			) {
61				// We have everything we need to ask for a login
62				var script = document.createElement( "script" );
63				script.setAttribute( "id", "wpcom_remote_login_validate" );
64				script.src = '/remote-login.php?wpcom_remote_login=validate'
65					+ '&wpcomid=' + data.wpcomid
66					+ '&token=' + encodeURIComponent( data.token )
67					+ '&host=' + window.location.protocol
68					+ '//' + window.location.hostname
69					+ '&postid=289'
70					+ '&is_singular=1';
71				document.body.appendChild( script );
72			}
73
74			return;
75		}
76
77		// Safari ITP, not logged in, so redirect
78		if ( data.msg === 'LOGIN-REDIRECT' ) {
79			window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
80			return;
81		}
82
83		// Safari ITP, storage access failed, remove the request
84		if ( data.msg === 'LOGIN-REMOVE' ) {
85			var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
86			var style_zap = document.createElement( 'style' );
87			style_zap.type = 'text/css';
88			style_zap.appendChild( document.createTextNode( css_zap ) );
89			document.body.appendChild( style_zap );
90
91			var e = document.getElementById( 'wpcom_request_access_iframe' );
92			e.parentNode.removeChild( e );
93
94			document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
95
96			return;
97		}
98
99		// Safari ITP
100		if ( data.msg === 'REQUEST_ACCESS' ) {
101			console.log( 'request access: safari' );
102
103			// Check ITP iframe enable/disable knob
104			if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
105				return;
106			}
107
108			// If we are in a "private window" there is no ITP.
109			var private_window = false;
110			try {
111				var opendb = window.openDatabase( null, null, null, null );
112			} catch( e ) {
113				private_window = true;
114			}
115
116			if ( private_window ) {
117				console.log( 'private window' );
118				return;
119			}
120
121			var iframe = document.createElement( 'iframe' );
122			iframe.id = 'wpcom_request_access_iframe';
123			iframe.setAttribute( 'scrolling', 'no' );
124			iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
125			iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
126
127			var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
128
129			var style = document.createElement( 'style' );
130			style.type = 'text/css';
131			style.id = 'wpcom_request_access_styles';
132			style.appendChild( document.createTextNode( css ) );
133			document.body.appendChild( style );
134
135			document.body.appendChild( iframe );
136		}
137
138		if ( data.msg === 'DONE' ) {
139			wpcom_remote_login_final_cleanup();
140		}
141	}
142}, false );
143
144// Inject the remote login iframe after the page has had a chance to load
145// more critical resources
146window.addEventListener( "DOMContentLoaded", function( e ) {
147	var iframe = document.createElement( "iframe" );
148	iframe.style.display = "none";
149	iframe.setAttribute( "scrolling", "no" );
150	iframe.setAttribute( "id", "wpcom_remote_login_key" );
151	iframe.src = "https://r-login.wordpress.com/remote-login.php"
152		+ "?wpcom_remote_login=key"
153		+ "&origin=aHR0cHM6Ly9jbG91ZG5jb2RlLmJsb2c%3D"
154		+ "&wpcomid=113813661"
155		+ "&time=" + Math.floor( Date.now() / 1000 );
156	document.body.appendChild( iframe );
157}, false );
158</script>
158
159<link rel='dns-prefetch' href='//s1.wp.com' />
160<link rel='dns-prefetch' href='//s2.wp.com' />
161<link rel='dns-prefetch' href='//widgets.wp.com' />
162<link rel='dns-prefetch' href='//wordpress.com' />
163<link rel='dns-prefetch' href='//s0.wp.com' />
164<link rel='dns-prefetch' href='//af.pubmine.com' />
165<link rel="alternate" type="application/rss+xml" title="cloudncode &raquo; Feed" href="https://cloudncode.blog/feed/" />
166<link rel="alternate" type="application/rss+xml" title="cloudncode &raquo; Comments Feed" href="https://cloudncode.blog/comments/feed/" />
167<link rel="alternate" type="application/rss+xml" title="cloudncode &raquo; Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS Comments Feed" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/feed/" />
168	
168<script type="text/javascript">
169		/* <![CDATA[ */
170		function addLoadEvent(func) {
171			var oldonload = window.onload;
172			if (typeof window.onload != 'function') {
173				window.onload = func;
174			} else {
175				window.onload = function () {
176					oldonload();
177					func();
178				}
179			}
180		}
181		/* ]]> */
182	</script>
182
183	<style id="wp-img-auto-sizes-contain-inline-css">
184img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
185/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
186</style>
187<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/_static/??/wp-content/mu-plugins/widgets/eu-cookie-law/templates/style.css,/wp-content/mu-plugins/likes/jetpack-likes.css?m=1743883414j&cssminify=yes' type='text/css' media='all' />
188<style id="wp-emoji-styles-inline-css">
189
190	img.wp-smiley, img.emoji {
191		display: inline !important;
192		border: none !important;
193		box-shadow: none !important;
194		height: 1em !important;
195		width: 1em !important;
196		margin: 0 0.07em !important;
197		vertical-align: -0.1em !important;
198		background: none !important;
199		padding: 0 !important;
200	}
201/*# sourceURL=wp-emoji-styles-inline-css */
202</style>
203<style id="wp-block-library-inline-css">
204:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;
204--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
205
206/*# sourceURL=/wp-content/plugins/gutenberg-core/v24.0.0/build/styles/block-library/common.min.css */
207</style>
208<style id="wp-block-library-inline-css-extra">
209.has-text-align-justify {
210	text-align:justify;
211}
212.has-text-align-justify{text-align:justify;}
213/*# sourceURL=wp-block-library-inline-css */
214</style>
215
216<style id="classic-theme-styles-inline-css">
217.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
218/*# sourceURL=/wp-content/plugins/gutenberg-core/v24.0.0/build/styles/block-library/classic.min.css */
219</style>
220
221<style id="coblocks-inline-format-inline-css">
222.uppercase{text-transform:uppercase}
223/*# sourceURL=coblocks-inline-format-inline-css */
224</style>
225<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??/wp-content/mu-plugins/core-compat/wp-mediaelement.css,/wp-content/mu-plugins/wpcom-bbpress-premium-themes.css?m=1432920480j&cssminify=yes' type='text/css' media='all' />
226<style id="global-styles-inline-css">
227:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,
2270) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}
227.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
228/*# sourceURL=global-styles-inline-css */
229</style>
230
231<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.css?m=1785880900i&cssminify=yes' type='text/css' media='all' />
232<link rel='stylesheet' id='verbum-gutenberg-css-css' href='https://widgets.wp.com/verbum-block-editor/block-editor.css?ver=1771405207' media='all' />
233<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/_static/??-eJx9jVEOwjAMQy9EicYQiA/EWbYSSqBNqjXV4PYEkGCAxJ/9EtswZueFFVkhVZdjDcQFvKRkyEU6o7nyReZGZkDAomTp8hLPw6RTj5isIdce8JKl8h6KXiP+PE7GB+yjBJPhsfy2/0IBxUXxnZLwh3GH2NFwj+7Stlkvm9WmbdvF6QaGbl0a&cssminify=yes' type='text/css' media='all' />
234<style id="jetpack-global-styles-frontend-style-inline-css">
235:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
236/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
237</style>
238<link crossorigin='anonymous' rel='stylesheet' id='all-css-18-1' href='/_static/??-eJyNjcsKAjEMRX/ImhmUURfip0inLW3GNCmmQfx7H7gRN+7ugcs5cGsuCPfEHaq5RpaRFZbUmw+XD4MaQ5VolBS0+GuKPsb7eyLndVBdwd+iM3KA2ZAiqAT05Eiy6Bf8KHtJ9RkvW8gks6fX4VSP426/mQ7TMA7LA38FSqA=&cssminify=yes' type='text/css' media='all' />
239<script id="wpcom-actionbar-placeholder-js-extra">
240var actionbardata = {"siteID":"113813661","postID":"289","siteURL":"https://cloudncode.blog","xhrURL":"https://cloudncode.blog/wp-admin/admin-ajax.php","nonce":"ba836f7ed5","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&amp;ver=20211021","shortlink":"https://wp.me/p7Hy8d-4F","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Copied to clipboard."}};
241//# sourceURL=wpcom-actionbar-placeholder-js-extra
242</script>
vendor: 1 bytes, line 242
242
243<script id="jetpack-mu-wpcom-settings-js-before">
244var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
245//# sourceURL=jetpack-mu-wpcom-settings-js-before
246</script>
vendor: 1 bytes, line 246
246
247<script crossorigin='anonymous' type='text/javascript'  src='/_static/??/wp-content/js/rlt-proxy.js,/wp-content/blog-plugins/wordads-classes/js/cmp/v2/cmp-non-gdpr.js?m=1780567333j'></script>
vendor: 1 bytes, line 247
247
248<script id="rlt-proxy-js-after">
249	rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
250//# sourceURL=rlt-proxy-js-after
251</script>
251
252<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://cloudncode.wordpress.com/xmlrpc.php?rsd" />
253<meta name="generator" content="WordPress.com" />
254<link rel="canonical" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" />
255<link rel='shortlink' href='https://wp.me/p7Hy8d-4F' />
256<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&amp;url=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&amp;for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&amp;url=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&amp;for=wpcom-auto-discovery" />
257<!-- Jetpack Open Graph Tags -->
258<meta property="og:type" content="article" />
259<meta property="og:title" content="Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS" />
260<meta property="og:url" content="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" />
261<meta property="og:description" content="This is the third article in the series on What should you log in an application and how to avoid having 24×7 support looking at them? In the previous article we talked about how to setup Elastic S…" />
262<meta property="article:published_time" content="2017-02-06T12:56:42+00:00" />
263<meta property="article:modified_time" content="2017-05-24T14:22:56+00:00" />
264<meta property="og:site_name" content="cloudncode" />
265<meta property="og:image" content="https://i0.wp.com/cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?fit=1200%2C788&#038;ssl=1" />
266<meta property="og:image:width" content="1200" />
267<meta property="og:image:height" content="788" />
268<meta property="og:image:alt" content="" />
269<meta property="og:locale" content="en_US" />
270<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
271<meta name="twitter:creator" content="@maingified" />
272<meta name="twitter:site" content="@maingified" />
273<meta name="twitter:text:title" content="Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS" />
274<meta name="twitter:image" content="https://i0.wp.com/cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?fit=1200%2C788&#038;ssl=1&#038;w=640" />
275<meta name="twitter:card" content="summary_large_image" />
276
277<!-- End Jetpack Open Graph Tags -->
278<link rel="shortcut icon" type="image/x-icon" href="https://s1.wp.com/i/favicon.ico?m=1713425267i" sizes="16x16 24x24 32x32 48x48" />
279<link rel="icon" type="image/x-icon" href="https://s1.wp.com/i/favicon.ico?m=1713425267i" sizes="16x16 24x24 32x32 48x48" />
280<link rel="apple-touch-icon" href="https://s2.wp.com/i/webclip.png?m=1713868326i" />
281<link rel="search" type="application/opensearchdescription+xml" href="https://cloudncode.blog/osd.xml" title="cloudncode" />
282<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
283<meta name="description" content="This is the third article in the series on What should you log in an application and how to avoid having 24×7 support looking at them? In the previous article we talked about how to setup Elastic Search and Kibana, in this article we’ll learn how to properly send logs from your application servers to&hellip;" />
284<script>
285var wa_client = {}; wa_client.cmd = []; wa_client.config = { 'blog_id': 113813661, 'blog_language': 'en', 'is_wordads': true, 'hosting_type': 1, 'afp_account_id': null, 'afp_host_id': 5038568878849053, 'theme': 'pub/expound', '_': { 'title': 'Advertisement', 'privacy_settings': 'Privacy Settings' }, 'formats': [ 'inline', 'belowpost', 'bottom_sticky', 'sidebar_sticky_right', 'sidebar', 'top', 'shortcode', 'gutenberg_rectangle', 'gutenberg_leaderboard', 'gutenberg_mobile_leaderboard', 'gutenberg_skyscraper' ] };
286</script>
286
287		
287<script type="text/javascript">
288
289			window.doNotSellCallback = function() {
290
291				var linkElements = [
292					'a[href="https://wordpress.com/?ref=footer_blog"]',
293					'a[href="https://wordpress.com/?ref=footer_website"]',
294					'a[href="https://wordpress.com/?ref=vertical_footer"]',
295					'a[href^="https://wordpress.com/?ref=footer_segment_"]',
296				].join(',');
297
298				var dnsLink = document.createElement( 'a' );
299				dnsLink.href = 'https://wordpress.com/advertising-program-optout/';
300				dnsLink.classList.add( 'do-not-sell-link' );
301				dnsLink.rel = 'nofollow';
302				dnsLink.style.marginLeft = '0.5em';
303				dnsLink.textContent = 'Do Not Sell or Share My Personal Information';
304
305				var creditLinks = document.querySelectorAll( linkElements );
306
307				if ( 0 === creditLinks.length ) {
308					return false;
309				}
310
311				Array.prototype.forEach.call( creditLinks, function( el ) {
312					el.insertAdjacentElement( 'afterend', dnsLink );
313				});
314
315				return true;
316			};
317
318		</script>
318
319					<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s1.wp.com/?custom-css=1&#038;csblog=7Hy8d&#038;cscache=6&#038;csrev=9" />
320			
320<script type="text/javascript">
321	window.google_analytics_uacct = "UA-52447-2";
322</script>
322
323
324<script type="text/javascript">
325	var _gaq = _gaq || [];
326	_gaq.push(['_setAccount', 'UA-52447-2']);
327	_gaq.push(['_gat._anonymizeIp']);
328	_gaq.push(['_setDomainName', 'none']);
329	_gaq.push(['_setAllowLinker', true]);
330	_gaq.push(['_initData']);
331	_gaq.push(['_trackPageview']);
332
333	(function() {
334		var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
335		ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
336		(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(ga);
337	})();
338</script>
338
339<style id="jetpack-subscriptions-inline-css">
340#subscribe-email input {
341	width: 95%;
342}
343
344.comment-subscription-form {
345	margin-bottom: 1em;
346}
347
348.comment-subscription-form .subscribe-label {
349	display: inline !important;
350}
351
352/*
353Text meant only for screen readers.
354Provides support for themes that do not bundle this CSS yet.
355@see https://make.wordpress.org/accessibility/2015/02/09/hiding-text-for-screen-readers-with-wordpress-core/
356***********************************/
357.screen-reader-text {
358	border: 0;
359	clip-path: inset(50%);
360	height: 1px;
361	margin: -1px;
362	overflow: hidden;
363	padding: 0;
364	position: absolute !important;
365	width: 1px;
366	overflow-wrap: normal !important;
367}
368
369/*# sourceURL=/wp-content/mu-plugins/jetpack-plugin/sun/_inc/build/subscriptions/subscriptions.min.css */
370</style>
371
372</head>
373
374<body class="wp-singular post-template-default single single-post postid-289 single-format-standard custom-background wp-theme-pubexpound customizer-styles-applied jetpack-reblog-enabled">
375<div id="page" class="hfeed site">
376		<header id="masthead" class="site-header" role="banner">
377		<div class="site-branding">
378			<a href="https://cloudncode.blog/" class="wrapper-link">
379				<div class="site-title-group">
380					<h1 class="site-title"><a href="https://cloudncode.blog/" title="cloudncode" rel="home">cloudncode</a></h1>
381					<h2 class="site-description">Cloud Computing and the Code that Powers it</h2>
382				</div>
383			</a>
384		</div>
385
386		<nav id="site-navigation" class="navigation-main" role="navigation">
387			<h1 class="menu-toggle">Menu</h1>
388			<div class="screen-reader-text skip-link"><a href="#content" title="Skip to content">Skip to content</a></div>
389
390			<div class="menu-primary-container"><ul id="menu-primary" class="menu"><li id="menu-item-12" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-12"><a href="/">Home</a></li>
391<li id="menu-item-420" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-420"><a href="https://cloudncode.blog/category/serverless-design/">Serverless Programming</a></li>
392<li id="menu-item-179" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-179"><a href="https://cloudncode.blog/category/cloud/">Cloud Related Articles</a></li>
393<li id="menu-item-180" class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-180"><a href="https://cloudncode.blog/category/code/">Code Related Articles</a></li>
394<li id="menu-item-13" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-13"><a href="https://cloudncode.blog/about/">About</a></li>
395<li id="menu-item-14" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-14"><a href="https://cloudncode.blog/contact/">Contact</a></li>
396</ul></div>					</nav><!-- #site-navigation -->
397	</header><!-- #masthead -->
398
399	<div id="main" class="site-main">
400
401	<div id="primary" class="content-area">
402		<div id="content" class="site-content" role="main">
403
404		
405			
406<article id="post-289" class="post-289 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud category-code category-serverless-design tag-elasticsearch tag-elasticsearch-on-aws tag-getting-kinesis-firehose-to-put-data-in-elastic-search tag-getting-started-with-kinesis-in-aws tag-how-to-handle-terrabyte-level-log-data-in-aws tag-how-to-horizontally-scale-log-storage-on-aws tag-kinesis-firehose tag-using-aws-lambda-with-kinesis">
407	<header class="entry-header">
408		<h1 class="entry-title">Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS</h1>
409
410		<div class="entry-meta">
411			<a class="author" rel="author" href="https://cloudncode.blog/author/maingi4/">Sumit Maingi</a> / <a class="entry-date" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/">February 6, 2017</a>		</div><!-- .entry-meta -->
412	</header><!-- .entry-header -->
413
414	<div class="entry-content">
415		<p>This is the third article in the series on <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/">What should you log in an application and how to avoid having 24×7 support looking at them?</a> In the previous article we talked about <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">how to setup Elastic Search and Kibana</a>, in this article we’ll learn how to properly send logs from your application servers to Elastic search and do some processing in between and after we are done, we’ll have a system ready which can handle terrabytes of log data with rare human intervention requirements.<span id="more-289"></span></p>
416<h2 id="so-what-is-this-kinesis-thingie">So… what is this Kinesis thingie?</h2>
417<p>Kinesis is an AWS data streaming service whose job is to deliver data reliably from one end to the other while adding extensions so as to make processing data easy and <em>scalable</em>. It is divided into two <em>main</em> sub-products:</p>
418<ol>
419<li><a href="https://aws.amazon.com/kinesis/streams/">
419Kinesis Streams</a>: Open ended data streams where you have data producers and consumers, you can define your own data consumers, this one does not scale on its own and you have to define the number of shards (capacity) you want from it beforehand.</li>
420<li><a href="https://aws.amazon.com/kinesis/firehose/">Kinesis Firehose</a>: This is same as above with the difference that it scales automatically and although data producers can be anything, data consumers are pre-defined, at the time of writing the article, valid consumers were:a. <a href="https://aws.amazon.com/s3/">Amazon S3</a>b. <a href="https://aws.amazon.com/elasticsearch-service/">Amazon Elastic Search service</a>c. <a href="https://aws.amazon.com/redshift/">Amazon Redshift</a>d. <a href="https://aws.amazon.com/kinesis/analytics/">Amazon Kinesis Analytics service</a></li>
421</ol>
422<p>Because we don’t want to worry about scaling and are interested in putting the data in AWS ES service anyway, we are going to go with Kinesis Firehose. If there is still some confusion about what this is for, it will get cleared up as we try setting stuff up below.</p>
423<h2 id="why-kinesis-why-not-simply-send-stuff-to-elastic-search-directly">Why Kinesis? Why not simply send stuff to Elastic Search directly?</h2>
424<h3 id="pt1-because-logs-also-contain-blobs">Pt#1: Because logs also contain blobs</h3>
425<p>Typically logs contains exception logs, server events, application events (Trace, debug info’s and request/responses) all these are necessary for good analysis of your data and is key to correctly and swiftly analyzing log data for diagnostic or analytical purposes. Now log lines (line = 1 entry of whatever type) which contains potentially huge chunks of unstructured data like request/responses which is never really searched upon and only looked at for diagnosing bugs, are actually increasing the packet size of what Elastic search is crunching across nodes. Even if the field is marked for no indexing (see here for <a href="https://www.elastic.co/guide/en/elasticsearch/guide/current/mapping-intro.html#_index_2">ES options on field indexing</a>) the data will still play a role in cluster health in the following scenarios:</p>
426<ol>
427<li>Shard redistribution (common) happens when you add a node to ES, or a node goes offline and replica shards get activated and the load gets redistributed across nodes for balancing, when the node comes back up the redistribution will begin again causing cascading failures (shudder) in case the size of the shards are too large</li>
428<li>Shard resizing (less often), happens when you change the number of shards for an index</li>
429<li>The Node coordinating the search result across various shards across different machines needs to collate large amounts of data size-wise in order to return the merged result, this is very resource intensive and may cause intermittent node outage, resulting in shard redistribution which again might cause the same thing due to large shard sizes, cascading failure is back!</li>
430</ol>
431<p>The only way to get away from the above stated problems would therefore be to throw GB’s of more RAM and storage coupled with double digit ethernet bandwidth with double the required nodes, if you shuddered at the thought of this statement then no, don’t just store everything on ES.</p>
432<h3 id="pt2-because-lots-of-logs-are-unprocessed">Pt#2: Because lots of logs are unprocessed</h3>
433<p>Logs, a lot of times are unprocessed e.g. you might want to add geographical data (lat/long) to the user IP address that you collected (imagine heat maps in Kibana), server events which are just text lines and cannot be searched upon (you want to convert these to JSON with readable attributes before you shove it in ES), lots of other things. What you don’t want to do is do these processings on your web servers. Web server’s primary responsibility is to serve your customers the best and as fast as they can, doing these resource intensive work on each of your servers and subsequently making sure that all your services which make up your application are following the same standard is just not worth it.</p>
434<p>A good log data pipeline makes sure you have a scalable way to process these logs before they reach your log database so that your web servers are not effected while at the same time you have good searchable data that you can build amazing, actionable dashboards and alarms from.</p>
435<h3 id="hmm-ok-then-what-should-i-be-storing-in-es-now">Hmm.. ok.. then what should I be storing in ES now?</h3>
436<p>Elastic search as its name implies is brilliant at searching and even aggregating results across a humongous dataset while giving you near perfect horizontal scalability, use it to store fields that you search upon, aggregate upon. If you need to search for something in the request or response, you should be taking out that value and putting that in a separate field.</p>
437<h3 id="ok-ill-buy-that-where-do-you-suppose-i-put-unstructured-data-in-then">Ok, I’ll buy that, where do you suppose I put unstructured data in then?</h3>
438<p>Unstructured blobs (generally request/responses) should be put in storage optimized for large blob storage, which are essentially disks, in the AWS world, you have two options <a href="https://aws.amazon.com/s3/">S3</a> (http based file storage) and <a href="https://aws.amazon.com/efs/">EFS</a> (made for cloud, network drive). In this article, I’ll be using S3 as the blob storage mechanism, simply because its easier to integrate and has easy auto-archiving strategies.</p>
439<p>In this guide we’ll send the request/response through Kinesis and just before stuff gets into Elastic search we’ll have Lambda functions massaging the log data, take out the blobs, put them into S3 and replace the log lines with clickable S3 links instead.</p>
440<h3 id="are-there-any-limitations-to-kinesis">Are there any limitations to Kinesis?</h3>
441<p>While I would love to say that Kinesis is the anti-Gandalf and anything can pass through, it does have the following limitations:</p>
442<ol>
443<li>The maximum size of a record sent to Firehose, before base64-encoding, is 1000 KB (you can compress your request/response to meet this requirement)</li>
444<li>The PutRecordBatch (bulk insert into the stream) operation can take up to 500 records per call or 4 MB per call, whichever is smaller. This limit cannot be changed</li>
445<li>Data is buffered for a minimum of 60 seconds or 1 MB (whichever is hit first), which basically means if you don’t have enough data going through the stream you might have to wait for a minute for your records to show up, this might be irritating in development environments, for production this is fine</li>
446</ol>
447<p>There are other AWS account related limitations which are worth noting, here is a <a href="http://docs.aws.amazon.com/firehose/latest/dev/limits.html">link to AWS documentation</a> for the complete set of Kinesis limitations, quite a few of these limitations can be increased by a simple support ticket to AWS via the console.</p>
448<h2 id="enough-talk-lets-setup-the-log-data-pipeline-already">Enough talk, Let’s setup the log data pipeline already!</h2>
449<h3 id="setting-up-elastic-search">Setting up Elastic search</h3>
450<p>I have the elastic search (5.1) already setup using AWS Elastic search service, <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">see this article</a> where I take you step by step in how to do that.</p>
451<h3 id="setting-up-a-index-template-for-time-based-log-indexes">Setting up a index template for time-based log indexes</h3>
452<p>Now if you went through the above mentioned article to setup your ES you already know that you <em>should</em> have time-based indexes for logging in ES, however, in order to control the behavior of these auto-generated time based indexes you can put index templates in ES so that whenever a new index is created for the day/week/month etc it automatically inherits the traits from the template that we define.</p>
453<p>We will define 4 shards per index and one replica since in the test setup we have only one elastic search node, you might want to have at least 2 replica’s in production.</p>
454<pre class="brush: jscript; title: ; notranslate" title="">
455PUT _template/active-logs
456{
457&quot;template&quot;: &quot;active-logs-*&quot;,
458&quot;settings&quot;: {
459&quot;number_of_shards&quot;: 4,
460&quot;number_of_replicas&quot;: 1
461}
462}
463</pre>
464<p><em>There is a maintenance overhead for shards so you don’t want too many shards for an index however you also don’t want gargantuan shards which w
464ould overload a node’s resources while aggregating/transferring/replicating etc.</em></p>
465<h3 id="getting-lambda-methods-to-intercept-the-log-data-and-slicing-blobs-into-s3">Getting Lambda methods to intercept the log data and slicing blobs into S3</h3>
466<p>We will transform and put the request and response in a S3 bucket as discussed before, create the lambda function exactly as I demonstrated <a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/">in this post</a> and swap the <code>program.cs</code> file with this:</p>
467<pre class="brush: csharp; title: ; notranslate" title="">
468using System;
469using System.Collections.Generic;
470using System.IO;
471using System.Threading.Tasks;
472using Amazon.S3;
473using Amazon.S3.Model;
474
475namespace CSharpLambdaFunction
476{
477public class LambdaHandler
478{
479public static void Main()
480{
481Task.Run(async () =&gt;
482{
483Stream result;
484using (var stream = new FileStream($&quot;{Directory.GetCurrentDirectory()}\\testData.json&quot;, FileMode.Open))
485{
486result = await new LambdaHandler().LogHandler(stream);
487}
488using (var reader = new StreamReader(result))
489{
490var records = reader.ReadToEnd();
491
492Console.WriteLine(records);
493}
494}).GetAwaiter().GetResult();
495}
496
497public async Task LogHandler(Stream inputStream)
498{
499LogEntry entry;
500FirehoseInput input;
501using (var reader = new StreamReader(inputStream))
502{
503input = Deserialize(reader);
504}
505
506foreach (var record in input.records)
507{
508record.result = &quot;Ok&quot;;
509using (var memStream = new MemoryStream(Convert.FromBase64String(record.data)))
510{
511using (var reader = new StreamReader(memStream))
512{
513entry = Deserialize(reader);
514}
515}
516
517var reqTask = SendToS3AndGetLink(entry.Request);
518var resTask = SendToS3AndGetLink(entry.Response);
519
520entry.Request = await reqTask;
521entry.Response = await resTask;
522
523var data = ConvertToStream(entry);
524
525record.data = Convert.ToBase64String(data.ToArray());
526}
527
528return ConvertToStream(input);
529}
530
531private T Deserialize(TextReader reader)
532{
533var serializer = new Newtonsoft.Json.JsonSerializer();
534
535return (T)serializer.Deserialize(reader, typeof(T));
536}
537
538private MemoryStream ConvertToStream(T logEntry)
539{
540var serializer = new Newtonsoft.Json.JsonSerializer();
541
542var memStream = new MemoryStream();
543var writer = new StreamWriter(memStream);
544serializer.Serialize(writer, logEntry);
545
546writer.Flush();
547
548memStream.Position = 0;
549
550return memStream;
551}
552
553private Stream ConvertToStream(string value)
554{
555var memStream = new MemoryStream();
556var writer = new StreamWriter(memStream);
557
558writer.Write(value);
559writer.Flush();
560
561return memStream;
562}
563
564private async Task SendToS3AndGetLink(string value)
565{
566var s3 = new AmazonS3Client(Amazon.RegionEndpoint.USWest2); //define your own region here
567
568var putRequest = new PutObjectRequest();
569
570var key = Guid.NewGuid().ToString().Replace(&quot;-&quot;, string.Empty);
571putRequest.BucketName = GetBucketName();
572putRequest.Key = key;
573putRequest.InputStream = ConvertToStream(value);
574putRequest.ContentType = &quot;application/json&quot;;
575
576var response = await s3.PutObjectAsync(putRequest);
577
578var urlRequest = new GetPreSignedUrlRequest();
579urlRequest.BucketName = GetBucketName();
580urlRequest.Expires = DateTime.UtcNow.AddYears(2);
581urlRequest.Key = key;
582
583return s3.GetPreSignedURL(urlRequest);
584}
585
586private string GetBucketName()
587{
588return &quot;cloudncode-logs&quot;;
589}
590}
591
592public class FirehoseInput
593{
594public string invocationId { get; set; }
595public string deliveryStreamArn { get; set; }
596public string region { get; set; }
597public List records { get; set; }
598}
599public class FirehoseRecord
600{
601public string recordId { get; set; }
602public string result { get; set; }
603public string data { get; set; }
604}
605
606public class LogEntry
607{
608public string RequestId { get; set; }
609public string SessionId { get; set; }
610public string Timestamp { get; set; }
611public string UserId { get; set; }
612public string ServerIp { get; set; }
613public string Message { get; set; }
614public string Level { get; set; }
615public string Category { get; set; }
616public string Request { get; set; }
617public string Response { get; set; }
618}
619}
620</pre>
621<p>if you don’t want to learn how to do that right now, you can also choose to <a href="https://cloudncode.blog/wp-content/uploads/2017/02/logprocessor-lambda_kinesis.zip">download the source code</a> directly from here, but you will need Visual Studio Code to compile and run it. If you are simply downloading the source code, go the article I mentioned above and start from <em>&#8220;Step 6: Add AWS users and roles&#8221;</em>.</p>
622<p>Note that the source code here is slightly different, the Lambda article that I wrote talks about a general Lambda function, however, Kinesis firehose gives a specific input and expects a specific output for the transformation to work. For our general knowledge, here is what Kinesis firehose gives as input to our Lambda function:</p>
623<pre class="brush: jscript; title: ; notranslate" title="">
624{
625&quot;invocationId&quot; : &quot;cd6eb022-9528-42c6-82fb-e5e969ea7962&quot;,
626&quot;deliveryStreamArn&quot; : &quot;arn:aws:firehose:us-west-2:522390xxx:deliverystream/logstream&quot;,
627&quot;region&quot; : &quot;us-west-2&quot;,
628&quot;records&quot; : [{
629&quot;recordId&quot; : &quot;49570122502395591010539641575388588252247858295703863298&quot;,
630&quot;approximateArrivalTimestamp&quot; : 1486135421633,
631&quot;data&quot; : &quot;&quot;
632}
633]
634}
635</pre>
636<p>And here is the output that it expects:</p>
637<pre class="brush: jscript; title: ; notranslate" title="">
638{
639&quot;invocationId&quot;: &quot;cd6eb022-9528-42c6-82fb-e5e969ea7962&quot;,
640&quot;deliveryStreamArn&quot;: &quot;arn:aws:firehose:us-west-2:522390xxx:deliverystream/logstream&quot;,
641&quot;region&quot;: &quot;us-west-2&quot;,
642&quot;records&quot;: [
643{
644&quot;recordId&quot;: &quot;49570122502395591010539641575388588252247858295703863298&quot;,
645&quot;result&quot;: &quot;Ok&quot;, //Valid options are &quot;Ok&quot;, &quot;Dropped&quot; &amp; &quot;ProcessingFailed&quot;
646&quot;data&quot;: &quot;&quot;
647}
648]
649}
650</pre>
651<p>In the <code>record.result</code> field in the output above stating <code>Dropped</code> marks the result to be successful and indicates that the stream should stop further processing of the object and not send it any further.</p>
652<h3 id="visualizing-kinesis-data-flow">Visualizing Kinesis Data Flow</h3>
653<p>Before we move further and create the actual data stream, let’s make sure we have a picture in mind on how the Kinesis data stream flows from the origin (app servers/devices) to the destination which is Elasticsearch in our case.</p>
654<p>Consider the diagram below:</p>
655<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/data-flow.png?w=1700" alt="enter image description here" /></p>
656<ol>
657<li>Kinesis receives data from your app server or directly via a smartphone app</li>
658<li>The data stream invokes a Lambda function which takes out request and response and decompresses the incoming log if it was compressed from the origin (in our case for simplicity’s sake, we won’t compress it) and puts them in an S3 bucket and further proceeds to replace the same with URL’s which the Kibana users can click on later to view the req/res in their browsers</li>
659<li>Lambda then returns the transformed log back to Kinesis which then puts them into the AWS Elasticsearch service</li>
660<li>In case any logs fail and all retries fail (retries are configurable), the failed logs are put in a configurable S3 bucket which can then be processed seperately, note that you can also dump every log into an S3 bucket along with the Elasticsearch service, however unless you mind not getting a fat bill from AWS I would dissuade against that</li>
661<li>Logs can then be viewed by the user using Kibana</li>
662</ol>
663<h3 id="setting-up-kinesis-firehose">Setting up Kinesis Firehose</h3>
664<p>Go to AWS Kinesis service page and then select Kinesis Firehose as the type of data stream, proceed to click on “Create Delivery Stream” as shown below:</p>
665<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-1.png?w=1700" alt="Create a delivery stream" /></p>
666<p>We want the data stream (Kinesis firehose) to send the data to our already setup AWS Elastic Search service, follow these steps:</p>
667<ol>
668<li>Choose Amazon Elasticsearch Service in the destination dropdown</li>
669<li>Give your stream a name, remember it will contain log data</li>
670<li>Choose your ES domain and type in <code>active-logs</code> in the index textbox, this name must correspond to the temaplate pattern that we had setup earlier</li>
671<li>Choose index rotation to OneDay, you can choose something different here as well depending upon the amount of traffic that you normally get in a day. A day is a safe choice if you are confused</li>
672<li>Put in <code>applicationEvent</code> in the type, this is the ES type in the index that will hold the documents, we want this stream to put in application events for now</li>
673<li>Setup to choose to backup your failed documents in S3, you can choose the safer option to backup everything as well, remember S3 charges per PUT request, this is more of a <em>what you can afford to do</em> kind of a question</li>
674<li>Create a new S3 bucket here, the dropdown will guide on doing the same</li>
675<li>Click on <code>Next</code></li>
676</ol>
677<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-2.png?w=1700" alt="Creating Kinesis Firehose step 1" /></p>
678<p>In this step, we’ll configure our data stream to use the Lambda function that we had created earlier which will take away the request and responses and replace them with Url’s, simple steps here, just refer the image in case you get confused.<
678/p>
679<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-3.png?w=1700" alt="Creating Kinesis Firehose step 2" /></p>
680<ol>
681<li>Setup the <code>buffer size</code> to be <code>5MB</code> and <code>interval</code> to be <code>60 seconds</code>, basically Kinesis will collect and send in logs together to ES for efficiency sake and will do that as soon as any one of the two conditions are fulfilled, i.e. either the collected documents reach 5MB in size or 60 seconds have elapsed since the documents had started to get collected.</li>
682<li>Since the backup S3 bucket only contains failures, you don’t need compression here, you should compress docs if you are backing up everything to S3 though</li>
683<li>Leave the S3 data unencrypted, I prefer controlling access to S3 instead, makes life simpler</li>
684<li><code>Keep</code> the <code>error logging enabled</code>, it will help us in diagnosing any issues with our data stream</li>
685<li>Create an <code>IAM role</code> for your stream which will give it access to send backups to S3, invoke the Lambda function and put documents into ES, when you select “Create a new role” from the dropdown it guides you in creating the role with just enough permissions to get the stream to work</li>
686<li>Click on <code>Next</code><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/firehose-step-4.png?w=1700" alt="Creating Kinesis Firehose step 3" /></li>
687</ol>
688<h3 id="sending-data-to-kinesis">Sending data to Kinesis</h3>
689<p>Logs can be sent to Kinesis via a Kinesis agent or via the AWS SDK, the agent although a pretty good option is currently available only for Amazon Linux AMI with version 2015.09 or later, or Red Hat Enterprise Linux version 7 or later. The agent basically picks up files from your server and transmits them to Kinesis completely decoupling your app from Kinesis. To know more about this option go to this <a href="http://docs.aws.amazon.com/firehose/latest/dev/writing-with-agents.html">AWS documentation</a>.</p>
690<p>If you will be sending data via the AWS SDK from either a app server or a end-user device such as a smartphone you can check out the <a href="http://docs.aws.amazon.com/firehose/latest/dev/writing-with-sdk.html">AWS documentation</a> link for the same or you can download a simple utility that I built to quickly test things out, here is the <a href="https://cloudncode.blog/wp-content/uploads/2017/02/logprocessor-sampleloggenerator.zip">source code</a> for a sample tool that I made in dotnet core using Visual Studio Code, when you have the code opened, you can just change the <code>testData.json</code> file and put in whatever log request you want and run it. As always, change the region name in the code and make sure your AWS credentials file (<code>C:\Users\\.aws\credentials</code> for windows) contains valid credentials with permissions to send data to Kinesis. If you don’t have these credentials configured, you can learn how to do that from <a href="http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-quick-configuration">this link at AWS</a> site which covers other operating systems apart from windows as well.</p>
691<h3 id="seeing-the-results-on-elastic-search-through-kibana">Seeing the results on Elastic search (through Kibana)</h3>
692<p>At this point you should have:</p>
693<ol>
694<li>A running AWS Elastic search service with templates added for our indexes which will be created via Kinesis firehose</li>
695<li>A Lambda function which we will use to transform our logs and related S3 bucket where the request and responses will be moved into</li>
696<li>Kinesis Firehose all setup to receive data from the servers/devices</li>
697<li>Some test data inserted into Elastic search</li>
698</ol>
699<p>Now we will setup Kibana with a simple table example so that users can actually look at the logs that go into the system for analysis and action.</p>
700<p>Open Kibana, as a refresher you can find the URL from the same from your Elasticsearch AWS console page, refer image below:</p>
701<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-4.png?w=768" alt="AWS Elasticsearch console" /></p>
702<p>We need to add our index pattern into Kibana so that it knows that it needs to read the same, go to the management tab in Kibana and click on <code>Index Patterns</code></p>
703<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-1.png?w=1700" alt="Kibana add index" /></p>
704<p>Click on <code>Add New</code></p>
705<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-2.png?w=700" alt="Kibana add index 2" /></p>
706<p>Put in <code>active-logs-*</code> in the index pattern box, The ‘*’ here denotes that as new indexes are created based on time they will all be included in the search that you make, then proceed to click on <code>Create</code>.</p>
707<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-3.png?w=1700" alt="Kibana add index 3" /></p>
708<p>After the index is created, set it as default, this is optional of course, but as I had another index created while I wrote the ‘<a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">How to setup Elasticsearch</a>’ post, I didn’t want that to be my default.</p>
709<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-4.png?w=1700" alt="Kibana set index as default" /></p>
710<p>Go to the <code>Discover</code> tab and change your index to the correct one if not already (refer image), in case you don’t see your test data as shown in the image try changing the time from the top-right corner in case the data was added in the past as in my case, click on <code>Save</code> and fill in name for the query.</p>
711<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-5.png?w=1700" alt="Kibana creating query (Saved Search)" /></p>
712<p>Now we will add a dashboard where we will save add the data table which users can later come and look at.</p>
713<p>Go to the <code>Dashboard</code> tab, click on <code>Add</code> and go to the <code>Saved Search</code> tab in the <code>Add Panels</code> section. Here you should see the query that you had saved before, select that.</p>
714<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-6.png?w=1700" alt="Kibana dashboard step 1" /></p>
715<p>To save this dashboard, click on the <code>Save</code> button from the top bar, give your dashboard a name and proceed to click on <code>Save</code>.</p>
716<p><img title="" src="https://cloudncode.blog/wp-content/uploads/2017/02/kibana-step-7.png?w=1700" alt="Kibana dashboard step 2" /></p>
717<p>There you have it folks, users can now come to Kibana, look at logs, you can also setup other graphs like errors over time, user hits according to time of day or whatever you can think of.</p>
718<h3 id="short-note-on-s3-storage-mechanisms-archival-options">Short note on S3 storage mechanisms, archival options</h3>
719<p>S3 is a highly reliable redundant object storage service from AWS, along with simply providing space to store files, you can also:</p>
720<ol>
721<li>Manage security/access</li>
722<li>Setup events which can invoke Lambda functions to take some action (move data, setup alerts, send stuff into other databases/ queues for further processing etc)</li>
723<li>Manage archival of data by either deletion or moving to cheaper storage options like <a href="https://aws.amazon.com/glacier/">AWS Glacier</a></li>
724</ol>
725<p>If you want me to write on any of these topics specifically, let me know by commenting below.</p>
726<h2 id="whats-left-to-get-to-the-point-where-i-will-no-longer-need-24x7-support">What’s left to get to the point where I will no longer need 24&#215;7 support?</h2>
727<p>Now that we know <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/">what to log</a>, <a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/">how to get elastic search and kibana up</a> to view the logs and how to get data properly into elastic search <a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/">using AWS Lambda</a> to utilize S3 as blob storage, we need to now learn how to have:</p>
728<ol>
729<li>SNS integration with Pagerduty / emails for real-time alerts which<br />
730rings your phone for severity 1 issues</li>
731<li>Backup monitoring to monitor Elastic search to build a fool proof auto-alert system in case the primary alert mechanism begins to fail</li>
732</ol>
733<p><strong>Follow this blog/subscribe to email alerts (top of sidebar or below comments in mobile) so that you know 
733when any of these posts come about.</strong></p>
734<div id="atatags-1138136615-6ab58dd567449">
735		
735<script type="text/javascript">
736			__ATA = window.__ATA || {};
737			__ATA.cmd = window.__ATA.cmd || [];
738			__ATA.cmd.push(function() {
739				__ATA.initVideoSlot('atatags-1138136615-6ab58dd567449', {
740					sectionId: '1138136615',
741					format: 'inread'
742				});
743			});
744		</script>
744
745	</div><span id="wordads-inline-marker" style="display: none;"></span><div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-linkedin"><a rel="nofollow noopener noreferrer"
746				data-shared="sharing-linkedin-289"
747				class="share-linkedin sd-button share-icon"
748				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=linkedin"
749				target="_blank"
750				aria-labelledby="sharing-linkedin-289"
751				>
752				<span id="sharing-linkedin-289" hidden>Share on LinkedIn (Opens in new window)</span>
753				<span>LinkedIn</span>
754			</a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer"
755				data-shared="sharing-reddit-289"
756				class="share-reddit sd-button share-icon"
757				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=reddit"
758				target="_blank"
759				aria-labelledby="sharing-reddit-289"
760				>
761				<span id="sharing-reddit-289" hidden>Share on Reddit (Opens in new window)</span>
762				<span>Reddit</span>
763			</a></li><li class="share-tumblr"><a rel="nofollow noopener noreferrer"
764				data-shared="sharing-tumblr-289"
765				class="share-tumblr sd-button share-icon"
766				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=tumblr"
767				target="_blank"
768				aria-labelledby="sharing-tumblr-289"
769				>
770				<span id="sharing-tumblr-289" hidden>Share on Tumblr (Opens in new window)</span>
771				<span>Tumblr</span>
772			</a></li><li class="share-twitter"><a rel="nofollow noopener noreferrer"
773				data-shared="sharing-twitter-289"
774				class="share-twitter sd-button share-icon"
775				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=twitter"
776				target="_blank"
777				aria-labelledby="sharing-twitter-289"
778				>
779				<span id="sharing-twitter-289" hidden>Share on X (Opens in new window)</span>
780				<span>X</span>
781			</a></li><li class="share-facebook"><a rel="nofollow noopener noreferrer"
782				data-shared="sharing-facebook-289"
783				class="share-facebook sd-button share-icon"
784				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=facebook"
785				target="_blank"
786				aria-labelledby="sharing-facebook-289"
787				>
788				<span id="sharing-facebook-289" hidden>Share on Facebook (Opens in new window)</span>
789				<span>Facebook</span>
790			</a></li><li class="share-pinterest"><a rel="nofollow noopener noreferrer"
791				data-shared="sharing-pinterest-289"
792				class="share-pinterest sd-button share-icon"
793				href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=pinterest"
794				target="_blank"
795				aria-labelledby="sharing-pinterest-289"
796				>
797				<span id="sharing-pinterest-289" hidden>Share on Pinterest (Opens in new window)</span>
798				<span>Pinterest</span>
799			</a></li><li class="share-email"><a rel="nofollow noopener noreferrer"
800				data-shared="sharing-email-289"
801				class="share-email sd-button share-icon"
802				href="mailto:?subject=%5BShared%20Post%5D%20Scaling%20for%20TB%20level%20log%20data%20with%20Kinesis%20Firehose%20and%20lambda%20with%20Elastic%20Search%20on%20AWS&#038;body=https%3A%2F%2Fcloudncode.blog%2F2017%2F02%2F06%2Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%2F&#038;share=email"
803				target="_blank"
804				aria-labelledby="sharing-email-289"
805				data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="8446e5c825" data-email-sh
805are-track-url="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?share=email">
806				<span id="sharing-email-289" hidden>Email a link to a friend (Opens in new window)</span>
807				<span>Email</span>
808			</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-113813661-289-6ab58dd568836' data-src='//widgets.wp.com/likes/index.html?ver=20260924#blog_id=113813661&amp;post_id=289&amp;origin=cloudncode.wordpress.com&amp;obj_id=113813661-289-6ab58dd568836&amp;domain=cloudncode.blog' data-name='like-post-frame-113813661-289-6ab58dd568836' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div></div>			</div><!-- .entry-content -->
809
810	<footer class="entry-meta">
811		<a class="entry-date" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/">February 6, 2017</a> in <a href="https://cloudncode.blog/category/cloud/" rel="category tag">Cloud</a>, <a href="https://cloudncode.blog/category/code/" rel="category tag">Code</a>, <a href="https://cloudncode.blog/category/serverless-design/" rel="category tag">Serverless Design</a>. Tags: <a href="https://cloudncode.blog/tag/elasticsearch/" rel="tag">elasticsearch</a>, <a href="https://cloudncode.blog/tag/elasticsearch-on-aws/" rel="tag">elasticsearch on aws</a>, <a href="https://cloudncode.blog/tag/getting-kinesis-firehose-to-put-data-in-elastic-search/" rel="tag">Getting kinesis firehose to put data in elastic search</a>, <a href="https://cloudncode.blog/tag/getting-started-with-kinesis-in-aws/" rel="tag">getting started with Kinesis in AWS</a>, <a href="https://cloudncode.blog/tag/how-to-handle-terrabyte-level-log-data-in-aws/" rel="tag">how to handle terrabyte level log data in aws</a>, <a href="https://cloudncode.blog/tag/how-to-horizontally-scale-log-storage-on-aws/" rel="tag">how to horizontally scale log storage on aws</a>, <a href="https://cloudncode.blog/tag/kinesis-firehose/" rel="tag">Kinesis firehose</a>, <a href="https://cloudncode.blog/tag/using-aws-lambda-with-kinesis/" rel="tag">using AWS Lambda with Kinesis</a>
812			</footer><!-- .entry-meta -->
813</article><!-- #post-## -->
814			<div class="related-content">
815	<h3 class="related-content-title">Related posts</h3>
816	
817		<article id="post-181" class="post-181 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud tag-aws tag-aws-elastic-search-service-getting-started tag-cloud-computing tag-elastic-search-5-getting-started-aws tag-elastic-search-and-kibana tag-elastic-search-on-cloud-getting-started">
818
819						<div class="entry-thumbnail">
820				<a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=50&amp;h=50&amp;crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=50&amp;h=50&amp;crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=100&amp;h=100&amp;crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="286" data-permalink="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/es-5-1-step-8/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png" data-orig-size="1909,934" data-comments-opened="1" data-image-title="es-5-1-step-8" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/01/es-5-1-step-8.png?w=700" /></a>
821			</div>
822			
823			<header class="entry-header">
824				<h3 class="entry-title"><a href="https://cloudncode.blog/2017/01/16/how-to-set-up-elastic-search-kibana-on-aws/" rel="bookmark">How to Set up Elastic Search &amp; Kibana on AWS</a></h3>
825			</header><!-- .entry-header -->
826
827		</article>
828
829	
830		<article id="post-340" class="post-340 post type-post status-publish format-standard has-post-thumbnail hentry category-cloud category-code category-serverless-design tag-aws-lambda tag-aws-lambda-concurrency tag-aws-lambda-function-best-practices tag-aws-lambda-logging-best-practices tag-aws-lambda-performance-considerations tag-aws-lambda-stateless-design tag-choosing-the-right-aws-lambda-function-size tag-how-does-aws-lambda-scale tag-serverless-with-aws tag-streaming-aws-cloudwatch-logs-into-elasticsearch">
831
832						<div class="entry-thumbnail">
833				<a href="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=50&amp;h=50&amp;crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=50&amp;h=50&amp;crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=100&amp;h=100&amp;crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="372" data-permalink="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/2000px-orange_lambda-svg/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png" data-orig-size="2000,2090" data-comments-opened="1" data-image-title="2000px-orange_lambda-svg" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/03/2000px-orange_lambda-svg.png?w=700" /></a>
834			</div>
835			
836			<header class="entry-header">
837				<h3 class="entry-title"><a href="https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/" rel="bookmark">Best practices &#8211; AWS Lambda function</a></h3>
838			</header><!-- .entry-header -->
839
840		</article>
841
842	
843		<article id="post-219" class="post-219 post type-post status-publish format-standard has-post-thumbnail hentry category-code category-serverless-design tag-net-core tag-aws-kinesis tag-aws-lambda tag-aws-lambda-example-c tag-c tag-getting-started-with-visual-studio-code tag-getting-started-with-visual-studio-code-and-net-core tag-visual-studio-code">
844
845						<div class="entry-thumbnail">
846				<a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/"><img width="50" height="50" src="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=50&amp;h=50&amp;crop=1" class="attachment-expound-mini size-expound-mini wp-post-image" alt="" decoding="async" srcset="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=50&amp;h=50&amp;crop=1 50w, https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=100&amp;h=100&amp;crop=1 100w" sizes="(max-width: 50px) 100vw, 50px" data-attachment-id="226" data-permalink="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/dot-net-lambda-help/" data-orig-file="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png" data-orig-size="909,499" data-comments-opened="1" data-image-title="dot-net-lambda-help" data-image-description="" data-image-caption="" data-large-file="https://cloudncode.blog/wp-content/uploads/2017/01/dot-net-lambda-help.png?w=700" /></a>
847			</div>
848			
849			<header class="entry-header">
850				<h3 class="entry-title"><a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/" rel="bookmark">Getting started with writing and debugging AWS Lambda Function with Visual Studio Code</a></h3>
851			</header><!-- .entry-header -->
852
853		</article>
854
855	</div>
856
857				<nav role="navigation" id="nav-below" class="navigation-post">
858		<h1 class="screen-reader-text">Post navigation</h1>
859
860	
861		<div class="nav-previous"><a href="https://cloudncode.blog/2017/01/24/getting-started-with-writing-and-debugging-aws-lambda-function-with-visual-studio-code/" rel="prev"><span class="meta-nav">&larr;</span> Getting started with writing and debugging AWS Lambda Function with Visual Studio Code</a></div>		<div class="nav-next"><a href="https://cloudncode.blog/2017/02/13/net-core-web-api-lambda-performance/" rel="next">.Net Core Web API on AWS Lambda Performance <span class="meta-nav">&rarr;</span></a></div>
862	
863	</nav><!-- #nav-below -->
864	
865			
866	<div id="comments" class="comments-area">
867
868	
869			<h2 class="comments-title">
870			4 thoughts on &ldquo;<span>Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS</span>&rdquo;		</h2>
871
872		
873		<ol class="comment-list">
874				<li class="post pingback">
875		<p>Pingback: <a href="http://zerozone.com/mmp/aceinfoway/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws-/" class="url" rel="ugc external nofollow">Scaling for TB level log data with Kinesis Firehose and lambda with Elastic Search on AWS | Ace Infoway</a></p>
876	</li><!-- #comment-## -->
877	<li class="post pingback">
878		<p>Pingback: <a href="https://cloudncode.blog/2016/12/30/what-should-you-log-in-an-application-and-how-to-avoid-having-24x7-support-looking-at-them/" class="url" rel="ugc">What should you log in an application and how to avoid having 24&#215;
8787 support looking at them? | cloudncode</a></p>
879	</li><!-- #comment-## -->
880	<li class="comment even thread-even depth-1" id="li-comment-213">
881		<article id="comment-213" class="comment">
882			<footer>
883				<div class="comment-author vcard">
884					<img referrerpolicy="no-referrer" alt='Prune&#039;s avatar' src='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&#038;d=identicon&#038;r=R' srcset='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&#038;d=identicon&#038;r=R 1x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=60&#038;d=identicon&#038;r=R 1.5x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=80&#038;d=identicon&#038;r=R 2x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=120&#038;d=identicon&#038;r=R 3x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=160&#038;d=identicon&#038;r=R 4x' class='avatar avatar-40' height='40' width='40' loading='lazy' decoding='async' />					<cite class="fn">Prune</cite> <span class="says">says:</span>				</div><!-- .comment-author .vcard -->
885				
886				<div class="comment-meta commentmetadata">
887					<a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comment-213"><time datetime="2017-03-02T21:16:09+05:30">
888					March 2, 2017 at 9:16 pm					</time></a>
889									</div><!-- .comment-meta .commentmetadata -->
890			</footer>
891
892			<div class="comment-content"><p>I don&#8217;t understand. You&#8217;re saying &#8220;you don&#8217;t need everything or unprocessed logs inside ES&#8221;, but as far as I understant, Kinesis is just your &#8220;broker&#8221; and will ship everything into ES&#8230; right ?</p>
893<p id="comment-like-213" data-liked=comment-not-liked class="comment-likes comment-not-liked"><a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?like_comment=213&#038;_wpnonce=fd610c62b0" class="comment-like-link needs-login" rel="nofollow" data-blog="113813661"><span>Like</span></a><span id="comment-like-count-213" class="comment-like-feedback">Like</span></p>
894</div>
895
896			<div class="reply">
897			<a rel="nofollow" class="comment-reply-link" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?replytocom=213#respond" data-commentid="213" data-postid="289" data-belowelement="comment-213" data-respondelement="respond" data-replyto="Reply to Prune" aria-label="Reply to Prune">Reply</a>			</div><!-- .reply -->
898		</article><!-- #comment-## -->
899
900	<ul class="children">
901	<li class="comment byuser comment-author-prune998 odd alt depth-2" id="li-comment-214">
902		<article id="comment-214" class="comment">
903			<footer>
904				<div class="comment-author vcard">
905					<img referrerpolicy="no-referrer" alt='prune998&#039;s avatar' src='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&#038;d=identicon&#038;r=R' srcset='https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=40&#038;d=identicon&#038;r=R 1x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=60&#038;d=identicon&#038;r=R 1.5x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=80&#038;d=identicon&#038;r=R 2x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=120&#038;d=identicon&#038;r=R 3x, https://1.gravatar.com/avatar/a451bf6311a4d789407b80b5ff004eb4a803481a73277472d97e1c87942ffa47?s=160&#038;d=identicon&#038;r=R 4x' class='avatar avatar-40' height='40' width='40' loading='lazy' decoding='async' />					<cite class="fn">prune998</cite> <span class="says">says:</span>				</div><!-- .comment-author .vcard -->
906				
907				<div class="comment-meta commentmetadata">
908					<a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comment-214"><time datetime="2017-03-02T21:17:12+05:30">
909					March 2, 2017 at 9:17 pm					</time></a>
910									</div><!-- .comment-meta .commentmetadata -->
911			</footer>
912
913			<div class="comment-content"><p>Sorry, in fact the Lambda function could be extended to filter some logs&#8230;</p>
914<p id="comment-like-214" data-liked=comment-not-liked class="comment-likes comment-not-liked"><a href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?like_comment=214&#038;_wpnonce=e59dcbc8f6" class="comment-like-link needs-login" rel="nofollow" data-blog="113813661"><span>Like</span></a><span id="comment-like-count-214" class="comment-like-feedback">Liked by <a href="#" class="view-likers" data-like-count="1">1 person</a></span></p>
915</div>
916
917			<div class="reply">
918			<a rel="nofollow" class="comment-reply-link" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/?replytocom=214#respond" data-commentid="214" data-postid="289" data-belowelement="comment-214" data-respondelement="respond" data-replyto="Reply to prune998" aria-label="Reply to prune998">Reply</a>			</div><!-- .reply -->
919		</article><!-- #comment-## -->
920
921	</li><!-- #comment-## -->
922</ul><!-- .children -->
923</li><!-- #comment-## -->
924		</ol><!-- .comment-list -->
925
926		
927	
928	
929		<div id="respond" class="comment-respond">
930		<h3 id="reply-title" class="comment-reply-title">Leave a comment <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://cloudncode.blog/wp-comments-post.php" method="post" id="commentform" class="comment-form">
931
932
933<div class="comment-form__verbum transparent"></div><div class="verbum-form-meta"><input type='hidden' name='comment_post_ID' value='289' id='comment_post_ID' />
934<input type='hidden' name='comment_parent' id='comment_parent' value='0' />
935
936			<input type="hidden" name="highlander_comment_nonce" id="highlander_comment_nonce" value="3dff5f3ccb" />
937			<input type="hidden" name="verbum_show_subscription_modal" value="" /></div><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="b959427df8" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>&#916;<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="94"/>
937<script>
938document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() );
939</script>
939
940</p></form>	</div><!-- #respond -->
941	
942</div><!-- #comments -->
943
944		
945		</div><!-- #content -->
946	</div><!-- #primary -->
947
948	<div id="secondary" class="widget-area" role="complementary">
949				<aside id="rss-3" class="widget widget_rss"><h1 class="widget-title"><a class="rsswidget" href="https://cloudncode.blog/feed/" title="Syndicate this content"><img style="background: orange; color: white; border: none;" width="14" height="14" src="https://s.wordpress.com/wp-includes/images/rss.png?m=1354137473i" alt="RSS" /></a> <a class="rsswidget" href="https://cloudncode.blog" title="Cloud Computing and the Code that Powers it">RSS</a></h1><ul><li><a class='rsswidget' href='https://cloudncode.blog/2018/03/27/tidbit-getting-started-with-aws-lambda-function-using-nodejs/' title='This is a Tidbit, basically whenever you see Tidbit: in the title, it means I am going to simply throw some helpful code without explaining too much around it. What will you be able to do after going through this? Build &amp; deploy a REST API on AWS Lambda using nodeJS which uses AWS API…'>Tidbit: Getting started with AWS Lambda function using nodeJS</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2018/03/22/is-aws-lambda-expensive-to-run/' title='What the pricing pivots on, explain the pricing model for AWS Lambda, what you should look out for when it comes to pricing. After reading this article, you will be able to make a more educated decision on whether AWS Lambda (or serverless architecture in general) is right for you or not.'>Is AWS Lambda expensive to run?</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/11/07/make-your-first-iot-device-via-aws-iot-service-and-raspberry-pi/' title='IOT or Internet of Things is a hot topic these days, but more than that its exciting. I have been a developer for more than 10 years now and although I have designed and built applications that can handle millions of calls and analyze terabytes of data in real time, the most exciting thing I…'>Make your first IOT device via AWS IOT Service and Raspberry Pi</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/08/08/tidbit-api-gateway-as-a-s3-proxy-cloudformation-script-with-serverless-framework/' title='Quick helpful article to get you to setup an API gateway which acts as a S3 proxy using Cloudformation script. This is a Tidbit, basically whenever you see Tidbit: in the title, it means I am going to simply throw some helpful code without explaining too much around it. What you can use this for?…'>Tidbit: API Gateway as a S3 Proxy – CloudFormation script with Serverless Framework</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/08/01/serverless-ci-part-2-getting-started-with-aws-codepipeline/' title='This is the second and final part of Serverless CI. In the previous post I walked us through setting up an AWS CodeBuild job which allocates a server, runs your build job and then deallocates it on demand. In this post we are going to walkthrough creating an AWS CodePipeline job which would automate these…'>Serverless CI: Part 2 – Getting started with AWS CodePipeline</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/06/13/serverless-ci-part-1-getting-started-with-aws-codebuild/' title='On my road to Serverless and my mission to spend less time on maintenance and more on product development, I finally decided to let go of Jenkins and go for a Serverless version of CI. As I have everything else in AWS, AWS CodeBuild and CodePipeline seemed to be the perfect fit for my purpose.…'>Serverless CI: Part 1 – Getting started with AWS CodeBuild</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/05/15/why-serverless-architecture/' title='At Tavisca Solutions we started developing beats a few months ago and as the product discovery went to a stage when we wanted to begin with a fast iterative product which can quickly evolve to MVP (minimum viable product) stage we were faced with a rare opportunity. Every time we start a new product we…'>Why Serverless architecture?</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/04/14/getting-started-with-the-serverless-framework-on-aws/' title='I have been in hibernation for some time as I shifted jobs, moved houses and have been hellishly busy, so, my bad that I haven’t written an article in a while. In this article, I am going to introduce and help you get started with the serverless tool where we are going to create a…'>Getting Started with the Serverless framework on AWS</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/03/05/automation-tool-for-aws-lambda-sizing/' title='As a continuation to my post on best practices on AWS Lambda I wrote a tool which finds the right size of your existing Lambda function programmatically by continuously resizing it, firing test events at it until it finds the best size which would either: Save you the most money OR Give you the best…'>Automation tool for AWS Lambda sizing</a></li><li><a class='rsswidget' href='https://cloudncode.blog/2017/03/02/best-practices-aws-lambda-function/' title='In this post we are going to go through the best practices while building an AWS Lambda function. In order to go through this blog you should know what is it and 
949you should ideally have built at least a simple function around it so you have your bearings right, you can checkout this blog…'>Best practices – AWS Lambda function</a></li></ul></aside><aside id="blog_subscription-5" class="widget widget_blog_subscription jetpack_subscription_widget"><h1 class="widget-title"><label for="subscribe-field">Follow Blog via Email</label></h1>
950
951			<div class="wp-block-jetpack-subscriptions__container">
952			<form
953				action="https://subscribe.wordpress.com"
954				method="post"
955				accept-charset="utf-8"
956				data-blog="113813661"
957				data-post_access_level="everybody"
958				id="subscribe-blog"
959			>
960				<p>Enter your email address to follow this blog and receive notifications of new posts by email.</p>
961				<p id="subscribe-email">
962					<label
963						id="subscribe-field-label"
964						for="subscribe-field"
965						class="screen-reader-text"
966					>
967						Email Address:					</label>
968
969					<input
970							type="email"
971							name="email"
972							autocomplete="email"
973							
974							style="width: 95%; padding: 1px 10px"
975							placeholder="Email Address"
976							value=""
977							id="subscribe-field"
978							required
979						/>				</p>
980
981				<p id="subscribe-submit"
982									>
983					<input type="hidden" name="action" value="subscribe"/>
984					<input type="hidden" name="blog_id" value="113813661"/>
985					<input type="hidden" name="source" value="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/"/>
986					<input type="hidden" name="sub-type" value="widget"/>
987					<input type="hidden" name="redirect_fragment" value="subscribe-blog"/>
988					<input type="hidden" id="_wpnonce" name="_wpnonce" value="d0d76a4c49" />					<button type="submit"
989													class="wp-block-button__link"
990																	>
991						Follow					</button>
992				</p>
993			</form>
994						</div>
995			
996</aside><aside id="twitter_timeline-3" class="widget widget_twitter_timeline"><a class="twitter-timeline" data-height="400" data-theme="light" data-border-color="#e8e8e8" data-lang="EN" data-partner="jetpack" data-chrome="nofooter noborders transparent" href="https://twitter.com/maingified" href="https://twitter.com/maingified">My Tweets</a></aside>	</div><!-- #secondary -->
997	</div><!-- #main -->
998
999	<footer id="colophon" class="site-footer" role="contentinfo">
1000		<div class="site-info">
1001			<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
1002		</div><!-- .site-info -->
1003	</footer><!-- #colophon -->
1004</div><!-- #page -->
1005
1006<!--  -->
1007<script type="speculationrules">
1008{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/expound/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
1009</script>
vendor: 1 bytes, line 1009
1009
1010<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202639d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
vendor: 1 bytes, line 1010
1010
1011<script id="wpgroho-js-extra">
1012var WPGroHo = {"my_hash":""};
1013//# sourceURL=wpgroho-js-extra
1014</script>
vendor: 1 bytes, line 1014
1014
1015<script crossorigin='anonymous' type='text/javascript'  src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
1015
1016
1017	
1017<script>
1018		// Initialize and attach hovercards to all gravatars
1019		( function() {
1020			function init() {
1021				if ( typeof Gravatar === 'undefined' ) {
1022					return;
1023				}
1024
1025				if ( typeof Gravatar.init !== 'function' ) {
1026					return;
1027				}
1028
1029				Gravatar.profile_cb = function ( hash, id ) {
1030					WPGroHo.syncProfileData( hash, id );
1031				};
1032
1033				Gravatar.my_hash = WPGroHo.my_hash;
1034				Gravatar.init(
1035					'body',
1036					'#wp-admin-bar-my-account',
1037					{
1038						i18n: {
1039							'Edit your profile →': 'Edit your profile →',
1040							'View profile →': 'View profile →',
1041							'Contact': 'Contact',
1042							'Send money': 'Send money',
1043							'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
1044							'Gravatar not found.': 'Gravatar not found.',
1045							'Too Many Requests.': 'Too Many Requests.',
1046							'Internal Server Error.': 'Internal Server Error.',
1047							'Is this you?': 'Is this you?',
1048							'Claim your free profile.': 'Claim your free profile.',
1049							'Email': 'Email',
1050							'Home Phone': 'Home Phone',
1051							'Work Phone': 'Work Phone',
1052							'Cell Phone': 'Cell Phone',
1053							'Contact Form': 'Contact Form',
1054							'Calendar': 'Calendar',
1055						},
1056					}
1057				);
1058			}
1059
1060			if ( document.readyState !== 'loading' ) {
1061				init();
1062			} else {
1063				document.addEventListener( 'DOMContentLoaded', init );
1064			}
1065		} )();
1066	</script>
1066
1067
1068		<div style="display:none">
1069	<div class="grofile-hash-map-92c17cd84fa39613655d7b9be4e612f0">
1070	</div>
1071	<div class="grofile-hash-map-92c17cd84fa39613655d7b9be4e612f0">
1072	</div>
1073	</div>
1074		<!-- CCPA [start] -->
1075		
1075<script type="text/javascript">
1076			( function () {
1077
1078				var setupPrivacy = function() {
1079
1080					// Minimal Mozilla Cookie library
1081					// https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie/Simple_document.cookie_framework
1082					var cookieLib = window.cookieLib = {getItem:function(e){return e&&decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*"+encodeURIComponent(e).replace(/[\-\.\+\*]/g,"\\$&")+"\\s*\\=\\s*([^;]*).*$)|^.*$"),"$1"))||null},setItem:function(e,o,n,t,r,i){if(!e||/^(?:expires|max\-age|path|domain|secure)$/i.test(e))return!1;var c="";if(n)switch(n.constructor){case Number:c=n===1/0?"; expires=Fri, 31 Dec 9999 23:59:59 GMT":"; max-age="+n;break;case String:c="; expires="+n;break;case Date:c="; expires="+n.toUTCString()}return"rootDomain"!==r&&".rootDomain"!==r||(r=(".rootDomain"===r?".":"")+document.location.hostname.split(".").slice(-2).join(".")),document.cookie=encodeURIComponent(e)+"="+encodeURIComponent(o)+c+(r?"; domain="+r:"")+(t?"; path="+t:"")+(i?"; secure":""),!0}};
1083
1084					// Implement IAB USP API.
1085					window.__uspapi = function( command, version, callback ) {
1086
1087						// Validate callback.
1088						if ( typeof callback !== 'function' ) {
1089							return;
1090						}
1091
1092						// Validate the given command.
1093						if ( command !== 'getUSPData' || version !== 1 ) {
1094							callback( null, false );
1095							return;
1096						}
1097
1098						// Check for GPC. If set, override any stored cookie.
1099						if ( navigator.globalPrivacyControl ) {
1100							callback( { version: 1, uspString: '1YYN' }, true );
1101							return;
1102						}
1103
1104						// Check for cookie.
1105						var consent = cookieLib.getItem( 'usprivacy' );
1106
1107						// Invalid cookie.
1108						if ( null === consent ) {
1109							callback( null, false );
1110							return;
1111						}
1112
1113						// Everything checks out. Fire the provided callback with the consent data.
1114						callback( { version: 1, uspString: consent }, true );
1115					};
1116
1117					// Initialization.
1118					document.addEventListener( 'DOMContentLoaded', function() {
1119
1120						// Internal functions.
1121						var setDefaultOptInCookie = function() {
1122							var value = '1YNN';
1123							var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname;
1124							cookieLib.setItem( 'usprivacy', value, 365 * 24 * 60 * 60, '/', domain );
1125						};
1126
1127						var setDefaultOptOutCookie = function() {
1128							var value = '1YYN';
1129							var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname;
1130							cookieLib.setItem( 'usprivacy', value, 24 * 60 * 60, '/', domain );
1131						};
1132
1133						var setDefaultNotApplicableCookie = function() {
1134							var value = '1---';
1135							var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname;
1136							cookieLib.setItem( 'usprivacy', value, 24 * 60 * 60, '/', domain );
1137						};
1138
1139						var setCcpaAppliesCookie = function( applies ) {
1140							var domain = '.wordpress.com' === location.hostname.slice( -14 ) ? '.rootDomain' : location.hostname;
1141							cookieLib.setItem( 'ccpa_applies', applies, 24 * 60 * 60, '/', domain );
1142						}
1143
1144						var maybeCallDoNotSellCallback = function() {
1145							if ( 'function' === typeof window.doNotSellCallback ) {
1146								return window.doNotSellCallback();
1147							}
1148
1149							return false;
1150						}
1151
1152						// Look for usprivacy cookie first.
1153						var usprivacyCookie = cookieLib.getItem( 'usprivacy' );
1154
1155						// Found a usprivacy cookie.
1156						if ( null !== usprivacyCookie ) {
1157
1158							// If the cookie indicates that CCPA does not apply, then bail.
1159							if ( '1---' === usprivacyCookie ) {
1160								return;
1161							}
1162
1163							// CCPA applies, so call our callback to add Do Not Sell link to the page.
1164							maybeCallDoNotSellCallback();
1165
1166							// We're all done, no more processing needed.
1167							return;
1168						}
1169
1170						// We don't have a usprivacy cookie, so check to see if we have a CCPA applies cookie.
1171						var ccpaCookie = cookieLib.getItem( 'ccpa_applies' );
1172
1173						// No CCPA applies cookie found, so we'll need to geolocate if this visitor is from California.
1174						// This needs to happen client side because we do not have region geo data in our $SERVER headers,
1175						// only country data -- therefore we can't vary cache on the region.
1176						if ( null === ccpaCookie ) {
1177
1178							var request = new XMLHttpRequest();
1179							request.open( 'GET', 'https://public-api.wordpress.com/geo/', true );
1180
1181							request.onreadystatechange = function () {
1182								if ( 4 === this.readyState ) {
1183									if ( 200 === this.status ) {
1184
1185										// Got a geo response. Parse out the region data.
1186										var data = JSON.parse( this.response );
1187										var region      = data.region ? data.region.toLowerCase() : '';
1188										var ccpa_applies = ['california', 'colorado', 'connecticut', 'delaware', 'indiana', 'iowa', 'montana', 'new jersey', 'oregon', 'tennessee', 'texas', 'utah', 'virginia'].indexOf( region ) > -1;
1189										// Set CCPA applies cookie. This keeps us from having to make a geo request too frequently.
1190										setCcpaAppliesCookie( ccpa_applies );
1191
1192										// Check if CCPA applies to set the proper usprivacy cookie.
1193										if ( ccpa_applies ) {
1194											if ( maybeCallDoNotSellCallback() ) {
1195												// Do Not Sell link added, so set default opt-in.
1196												setDefaultOptInCookie();
1197											} else {
1198												// Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe.
1199												setDefaultOptOutCookie();
1200											}
1201										} else {
1202											// CCPA does not apply.
1203											setDefaultNotApplicableCookie();
1204										}
1205									} else {
1206										// Could not geo, so let's assume for now that CCPA applies to be safe.
1207										setCcpaAppliesCookie( true );
1208										if ( maybeCallDoNotSellCallback() ) {
1209											// Do Not Sell link added, so set default opt-in.
1210											setDefaultOptInCookie();
1211										} else {
1212											// Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe.
1213											setDefaultOptOutCookie();
1214										}
1215									}
1216								}
1217							};
1218
1219							// Send the geo request.
1220							request.send();
1221						} else {
1222							// We found a CCPA applies cookie.
1223							if ( ccpaCookie === 'true' ) {
1224								if ( maybeCallDoNotSellCallback() ) {
1225									// Do Not Sell link added, so set default opt-in.
1226									setDefaultOptInCookie();
1227								} else {
1228									// Failed showing Do Not Sell link as required, so default to opt-OUT just to be safe.
1229									setDefaultOptOutCookie();
1230								}
1231							} else {
1232								// CCPA does not apply.
1233								setDefaultNotApplicableCookie();
1234							}
1235						}
1236					} );
1237				};
1238
1239				// Kickoff initialization.
1240				if ( window.defQueue && defQueue.isLOHP && defQueue.isLOHP === 2020 ) {
1241					defQueue.items.push( setupPrivacy );
1242				} else {
1243					setupPrivacy();
1244				}
1245
1246			} )();
1247		</script>
1247
1248
1249		<!-- CCPA [end] -->
1250		<div class="widget widget_eu_cookie_law_widget">
1251<div
1252	class="hide-on-button ads-active"
1253	data-hide-timeout="30"
1254	data-consent-expiration="180"
1255	id="eu-cookie-law"
1256	style="display: none"
1257>
1258	<form method="post">
1259		<input type="submit" value="Close and accept" class="accept" />
1260
1261		Privacy &amp; Cookies: This site uses cookies. By continuing to use this website, you agree to their use. <br />
1262To find out more, including how to control cookies, see here:
1263				<a href="https://automattic.com/cookies/" rel="nofollow">
1264			Cookie Policy		</a>
1265 </form>
1266</div>
1267</div>			<div id="actionbar" dir="ltr" style="display: none;"
1268				class="actnbr-pub-expound" role="region" aria-label="Site actions">
1269			<span class="actnbr-live" aria-live="polite"></span>
1270			<ul>
1271										<li class="actnbr-btn">
1272							<a class="actnbr-action actnbr-actn-comment" href="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/#comments">
1273								<svg class="actnbr-icon actnbr-icon-comment" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M18 4H6c-1.1 0-2 .9-2 2v12.9c0 .6.5 1.1 1.1 1.1.3 0 .5-.1.8-.3L8.5 17H18c1.1 0 2-.9 2-2V6c0-1.1-.9-2-2-2zm.5 11c0 .3-.2.5-.5.5H7.9l-2.4 2.4V6c0-.3.2-.5.5-.5h12c.3 0 .5.2.5.5v9z"/></svg>								<span>Comment							</span>
1274							</a>
1275						</li>
1276											<li class="actnbr-btn">
1277							<a class="actnbr-action actnbr-actn-reblog" href="" role="button">
1278								<svg class="actnbr-icon actnbr-icon-reusable-block" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M7 7.2h8.2L13.5 9l1.1 1.1 3.6-3.6-3.5-4-1.1 1 1.9 2.3H7c-.9 0-1.7.3-2.3.9-1.4 1.5-1.4 4.2-1.4 5.6v.2h1.5v-.3c0-1.1 0-3.5 1-4.5.3-.3.7-.5 1.2-.5zm13.8 4V11h-1.5v.3c0 1.1 0 3.5-1 4.5-.3.3-.7.5-1.3.5H8.8l1.7-1.7-1.1-1.1L5.9 17l3.5 4 1.1-1-1.9-2.3H17c.9 0 1.7-.3 2.3-.9 1.5-1.4 1.5-4.2 1.5-5.6z"/></svg><span>Reblog</span>
1279							</a>
1280						</li>
1281											<li class="actnbr-btn actnbr-hidden">
1282										<a class="actnbr-action actnbr-actn-follow " href="" role="button" aria-haspopup="dialog" aria-expanded="false">
1283				<svg class="actnbr-icon actnbr-icon-bell" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M17 11.5c0 1.353.17 2.368.976 3 .266.209.602.376 1.024.5v1H5v-1c.422-.124.757-.291 1.024-.5.806-.632.976-1.647.976-3V9c0-2.8 2.2-5 5-5s5 2.2 5 5v2.5ZM15.5 9v2.5c0 .93.066 1.98.515 2.897l.053.103H7.932a4.018 4.018 0 0 0 .053-.103c.449-.917.515-1.967.515-2.897V9c0-1.972 1.528-3.5 3.5-3.5s3.5 1.528 3.5 3.5Zm-5.492 9.008c0-.176.023-.346.065-.508h3.854A1.996 1.996 0 0 1 12 20c-1.1 0-1.992-.892-1.992-1.992Z"/></svg>				<span>Subscribe</span>
1284			</a>
1285			<a class="actnbr-action actnbr-actn-following  no-display" href="" role="button">
1286				<svg class="actnbr-icon actnbr-icon-check" width="20" height="20" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" style="fill: none" stroke="currentColor" stroke-width="1.5"><path d="M7 12L10 15L17 8" vector-effect="non-scaling-stroke"/></svg>				<span>Subscribed</span>
1287			</a>
1288									<div class="actnbr-popover actnbr-panel actnbr-notice" id="follow-bubble" role="dialog" aria-label="Subscribe to cloudncode">
1289								<div class="actnbr-follow-bubble">
1290								<div class="actnbr-panel__group"><a class="actnbr-menu__item actnbr-sitename" href="https://cloudncode.blog"><img loading='lazy' alt='' src='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png?m=1479929237i' srcset='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png 1x' class='avatar avatar-50' height='50' width='50' /><span class="actnbr-menu__label">cloudncode</span></a></div>																	<div class="actnbr-panel__group">
1291										<div class="actnbr-message no-display" aria-live="polite"></div>
1292										<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" class="no-display">
1293																							<div class="actnbr-follow-count">Join 112 other subscribers</div>
1294																							<input type="email" name="email" placeholder="Want updates? Enter your email" class="actnbr-email-field" aria-label="Email address" />
1295											<input type="hidden" name="action" value="subscribe" />
1296											<input type="hidden" name="blog_id" value="113813661" />
1297											<input type="hidden" name="source" value="https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" />
1298											<input type="hidden" name="sub-type" value="actionbar-follow" />
1299											<input type="hidden" id="_wpnonce" name="_wpnonce" value="d0d76a4c49" />											<button type="submit" class="actnbr-button">Sign me up</button>
1300										</form>
1301										<p class="actnbr-login-nudge">
1302											Have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fcloudncode.blog%252F2017%252F02%252F06%252Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%252F">Log in now.</a>										</p>
1303									</div>
1304																	</div>
1305							</div>
1306						</li>
1307									<li class="actnbr-ellipsis actnbr-hidden">
1308					<button type="button" class="actnbr-more-toggle" aria-haspopup="true" aria-expanded="false" aria-label="More options">
1309						<svg class="actnbr-icon actnbr-icon-more-horizontal" width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M11 13h2v-2h-2v2zm-6 0h2v-2H5v2zm12-2v2h2v-2h-2z"/></svg>					</button>
1310					<div class="actnbr-popover actnbr-menu" role="menu" aria-label="Site options">
1311						<div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-sitename" href="https://cloudncode.blog"><img loading='lazy' alt='' src='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png?m=1479929237i' srcset='https://s-ssl.wordpress.com/i/logo/wpcom-gray-white.png 1x' class='avatar avatar-50' height='50' width='50' /><span class="actnbr-menu__label">cloudncode</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-shortlink" href="https://wp.me/p7Hy8d-4F"><span class="actnbr-menu__label actnbr-shortlink__text">Copy shortlink</span><span class="actnbr-menu__icon actnbr-shortlink__icon"><svg class="actnbr-icon actnbr-icon-copy" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M5 4.5h11a.5.5 0 0 1 .5.5v11a.5.5 0 0 1-.5.5H5a.5.5 0 0 1-.5-.5V5a.5.5 0 0 1 .5-.5ZM3 5a2 2 0 0 1 2-2h11a2 2 0 0 1 2 2v11a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5Zm17 3v10.75c0 .69-.56 1.25-1.25 1.25H6v1.5h12.75a2.75 2.75 0 0 0 2.75-2.75V8H20Z"/></svg></span><span class="actnbr-menu__icon actnbr-shortlink__icon-copied"><svg class="actnbr-icon actnbr-icon-check" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" style="fill: none" stroke="currentColor" stroke-width="1.5"><path d="M7 12L10 15L17 8" vector-effect="non-scaling-stroke"/></svg></span></a><a role="menuitem" class="actnbr-menu__item actnbr-reader" href="https://wordpress.com/reader/blogs/113813661/posts/289"><span class="actnbr-menu__label">View post in Reader</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-subs" href="https://subscribe.wordpress.com/"><span class="actnbr-menu__label">Manage subscriptions</span></a><a role="menuitem" class="actnbr-menu__item actnbr-signup" href="https://wordpress.com/start/"><span class="actnbr-menu__label">Sign up</span></a><a role="menuitem" class="actnbr-menu__item actnbr-login" href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fcloudncode.blog%252F2017%252F02%252F06%252Fscaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws%252F"><span class="actnbr-menu__label">Log in</span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item flb-report" href="https://wordpress.com/abuse/?report_url=https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/" target="_blank" rel="noopener noreferrer"><span class="actnbr-menu__label">Report this content</span><span class="actnbr-menu__icon"><svg class="actnbr-icon actnbr-icon-external" width="18" height="18" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" fill="currentColor"><path d="M19.5 4.5h-7V6h4.44l-5.97 5.97 1.06 1.06L18 7.06v4.44h1.5v-7Zm-13 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2v-3H17v3a.5.5 0 0 1-.5.5h-10a.5.5 0 0 1-.5-.5v-10a.5.5 0 0 1 .5-.5h3V5.5h-3Z"/></svg></span></a></div><div class="actnbr-menu__group" role="group"><a role="menuitem" class="actnbr-menu__item actnbr-fold" href=""><span class="actnbr-menu__label">Collapse this bar</span></a></div>					</div>
1312				</li>
1313			</ul>
1314		</div>
1315		
1315<script>
1316window.addEventListener( "DOMContentLoaded", function () {
1317				var link = document.createElement( "link" );
1318				link.href = "\/wp-content\/mu-plugins\/jetpack-plugin\/sun\/jetpack_vendor\/automattic\/jetpack-newsletter\/src\/..\/build\/action-bar.css?ver=9e2715140ce0af99d983";
1319				link.rel = "stylesheet";
1320				document.head.appendChild( link );
1321
1322				var script = document.createElement( "script" );
1323				script.src = "\/wp-content\/mu-plugins\/jetpack-plugin\/sun\/jetpack_vendor\/automattic\/jetpack-newsletter\/src\/..\/build\/action-bar.js?ver=9e2715140ce0af99d983";
1324				document.body.appendChild( script );
1325			} );
1326</script>
1326
1327
1328	
1328<script type="text/javascript">
1329		window.WPCOM_sharing_counts = {"https://cloudncode.blog/2017/02/06/scaling-for-tb-level-log-data-with-kinesis-firehose-and-lambda-with-elastic-search-on-aws/":289};
1330	</script>
1330
1331						
1331<script crossorigin='anonymous' type='text/javascript'  src='/_static/??-eJzTLy/QTc7PK0nNK9EvyClNz8wr1i+uzCtJrMjITM/IAeKS1CJMEWP94uSizIISoOIM5/yiVL2sYh19yo1yKiotzvAKBvOpaaRzcEZiUQHQRPtcW0NTcwMjAxNLC9MsAGaiYW4='></script>
vendor: 1 bytes, line 1331
1331
1332<script type='text/javascript'>
1333	(function(){
1334		var corecss = document.createElement('link');
1335		var themecss = document.createElement('link');
1336		var corecssurl = "/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shCore.css?ver=3.0.9b";
1337		if ( corecss.setAttribute ) {
1338				corecss.setAttribute( "rel", "stylesheet" );
1339				corecss.setAttribute( "type", "text/css" );
1340				corecss.setAttribute( "href", corecssurl );
1341		} else {
1342				corecss.rel = "stylesheet";
1343				corecss.href = corecssurl;
1344		}
1345		document.head.appendChild( corecss );
1346		var themecssurl = "/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shThemeDefault.css?m=1363304414i&amp;ver=3.0.9b";
1347		if ( themecss.setAttribute ) {
1348				themecss.setAttribute( "rel", "stylesheet" );
1349				themecss.setAttribute( "type", "text/css" );
1350				themecss.setAttribute( "href", themecssurl );
1351		} else {
1352				themecss.rel = "stylesheet";
1353				themecss.href = themecssurl;
1354		}
1355		document.head.appendChild( themecss );
1356	})();
1357	SyntaxHighlighter.config.strings.expandSource = '+ expand source';
1358	SyntaxHighlighter.config.strings.help = '?';
1359	SyntaxHighlighter.config.strings.alert = 'SyntaxHighlighter\n\n';
1360	SyntaxHighlighter.config.strings.noBrush = 'Can\'t find brush for: ';
1361	SyntaxHighlighter.config.strings.brushNotHtmlScript = 'Brush wasn\'t configured for html-script option: ';
1362	SyntaxHighlighter.defaults['pad-line-numbers'] = false;
1363	SyntaxHighlighter.defaults['toolbar'] = false;
1364	SyntaxHighlighter.all();
1365
1366	// Infinite scroll support
1367	document.addEventListener( 'is.post-load', function () {
1368		SyntaxHighlighter.highlight();
1369	} );
1370</script>
1370
1371
1372	
1372<script type="text/javascript">
1373		(function () {
1374			var wpcom_reblog = {
1375				source: 'toolbar',
1376
1377				toggle_reblog_box_flair: function (obj_id, post_id) {
1378
1379					// Go to site selector. This will redirect to their blog if they only have one.
1380					const postEndpoint = `https://wordpress.com/post`;
1381
1382					// Ideally we would use the permalink here, but fortunately this will be replaced with the 
1383					// post permalink in the editor.
1384					const originalURL = `${ document.location.href }?page_id=${ post_id }`; 
1385					
1386					const url =
1387						postEndpoint +
1388						'?url=' +
1389						encodeURIComponent( originalURL ) +
1390						'&is_post_share=true' +
1391						'&v=5';
1392
1393					const redirect = function () {
1394						if (
1395							! window.open( url, '_blank' )
1396						) {
1397							location.href = url;
1398						}
1399					};
1400
1401					if ( /Firefox/.test( navigator.userAgent ) ) {
1402						setTimeout( redirect, 0 );
1403					} else {
1404						redirect();
1405					}
1406				},
1407			};
1408
1409			window.wpcom_reblog = wpcom_reblog;
1410		})();
1411	</script>
vendor: 1 bytes, line 1411
1411
1412<script crossorigin='anonymous' type='text/javascript'  src='/_static/??-eJx9jEsOwjAMBS9EcIuqdoU4i2msNKnzoXbI9ckO1AXbeTMPWjFrTkpJ4cnZmcLV+STQ8mHRilkZRUggdITK5n27BrnATxbrN/LWkQpQ7WvePRnGBkqxMCqd+J8f9nu3X5UqbZgs09HlR7yPyzxM0zyMS/gAPatD8g=='></script>
vendor: 1 bytes, line 1412
1412
1413<script id="wp-hooks-js" src="/wp-content/plugins/gutenberg-core/v24.0.0/build/scripts/hooks/index.min.js?m=1789650619i&#038;ver=f0f188028580e8dc1255"></script>
vendor: 1 bytes, line 1413
1413
1414<script id="wp-i18n-js" src="/wp-content/plugins/gutenberg-core/v24.0.0/build/scripts/i18n/index.min.js?m=1789650619i&#038;ver=1dfe7db3940c23ea9216"></script>
vendor: 1 bytes, line 1414
1414
1415<script id="wp-i18n-js-after">
1416wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } );
1417//# sourceURL=wp-i18n-js-after
1418</script>
vendor: 1 bytes, line 1418
1418
1419<script id="verbum-settings-js-before">
1420window.VerbumComments = {"Log in or provide your name and email to leave a reply.":"Log in or provide your name and email to leave a reply.","Log in or provide your name and email to leave a comment.":"Log in or provide your name and email to leave a comment.","Receive web and mobile notifications for posts on this site.":"Receive web and mobile notifications for posts on this site.","Name":"Name","Email (address never made public)":"Email (address never made public)","Website (optional)":"Website (optional)","Leave a reply. (log in optional)":"Leave a reply. (log in optional)","Leave a comment. (log in optional)":"Leave a comment. (log in optional)","Log in to leave a reply.":"Log in to leave a reply.","Log in to leave a comment.":"Log in to leave a comment.","Logged in via %s":"Logged in via %s","Log out":"Log out","Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.":"Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.","Email":"Email","(Address never made public)":"(Address never made public)","Instantly":"Instantly","Daily":"Daily","Reply":"Reply","Comment":"Comment","WordPress":"WordPress","Weekly":"Weekly","Notify me of new posts":"Notify me of new posts","Email me new posts":"Email me new posts","Email me new comments":"Email me new comments","Cancel":"Cancel","Write a comment...":"Write a comment...","Write a reply...":"Write a reply...","Website":"Website","Optional":"Optional","We'll keep you in the loop!":"We'll keep you in the loop!","Loading your comment...":"Loading your comment...","Discover more from":"Discover more from cloudncode","Subscribe now to keep reading and get access to the full archive.":"Subscribe now to keep reading and get access to the full archive.","Continue reading":"Continue reading","Never miss a beat!":"Never miss a beat!","Interested in getting blog post updates? Simply click the button below to stay in the loop!":"Interested in getting blog post updates? Simply click the button below to stay in the loop!","Enter your email address":"Enter your email address","Subscribe":"Subscribe","Comment sent successfully":"Comment sent successfully","Save my name, email, and website in this browser for the next time I comment.":"Save my name, email, and website in this browser for the next time I comment.","hovercardi18n":{"Edit your profile \u2192":"Edit your profile \u2192","View profile \u2192":"View profile \u2192","Contact":"Contact","Send money":"Send money","Gravatar not found.":"Gravatar not found.","This profile is private.":"This profile is private.","Too Many Requests.":"Too Many Requests.","Internal Server Error.":"Internal Server Error.","Sorry, we are unable to load this Gravatar profile.":"Sorry, we are unable to load this Gravatar profile."},"siteId":113813661,"postId":289,"mustLogIn":false,"requireNameEmail":true,"commentRegistration":false,"connectURL":"https://cloudncode.wordpress.com/public.api/connect/?action=request\u0026domain=cloudncode.blog","logoutURL":"https://cloudncode.wordpress.com/wp-login.php?action=logout\u0026_wpnonce=ba114a174e","homeURL":"https://cloudncode.blog/","subscribeToBlog":true,"subscribeToComment":true,"isJetpackCommentsLoggedIn":false,"jetpackUsername":"","jetpackUserId":0,"jetpackSignature":"","jetpackAvatar":"https://0.gravatar.com/avatar/?s=96\u0026amp;d=identicon\u0026amp;r=R","enableBlocks":true,"enableSubscriptionModal":true,"currentLocale":"en","isJetpackComments":false,"allowedBlocks":["core/paragraph","core/list","core/code","core/list-item","core/quote","core/image","core/embed","core/quote","core/code"],"embedNonce":"cb14ccd6fe","verbumBundleUrl":"/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/features/verbum-comments/dist/index.js","isRTL":false,"vbeCacheBuster":1789740582,"iframeUniqueId":0,"colorScheme":false}
1421//# sourceURL=verbum-settings-js-before
1422</script>
vendor: 1 bytes, line 1422
1422
1423<script data-wp-strategy="defer" defer id="verbum-dynamic-loader-js" src="/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/assets/dynamic-loader.js?m=1772522099i&#038;minify=false&#038;ver=8b5296f9195db92c42e4"></script>
vendor: 1 bytes, line 1423
1423
1424<script id="comment-like-js-extra">
1425var comment_like_text = {"loading":"Loading...","swipeUrl":"https://s2.wp.com/wp-content/mu-plugins/comment-likes/js/lib/swipe.js?m=1382645497i&amp;ver=20131008"};
1426//# sourceURL=comment-like-js-extra
1427</script>
vendor: 1 bytes, line 1427
1427
1428<script id="sharing-js-js-extra">
1429var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
1430//# sourceURL=sharing-js-js-extra
1431</script>
vendor: 1 bytes, line 1431
1431
1432<script id="jetpack-script-data-js-before">
1433window.JetpackScriptData = {"site":{"host":"wpcom","is_wpcom_platform":true}};
1434//# sourceURL=jetpack-script-data-js-before
1435</script>
vendor: 1 bytes, line 1435
1435
1436<script crossorigin='anonymous' type='text/javascript'  src='/_static/??-eJydUNFOwzAM/CHSgJi08YD4lMlL3M5t40Sxs25/T+i2qoCEJh4sWee789l2SsZFVmS1oZg0lo5YrIshVMiMNKDY/gfQ9PJkV0I9Yqi0VA4WzykW9l8ShhN1oBT5Mb4MlKo/D6aNrohp6XwXErux+O9JMqbx0gT65b46o0dN4AZToSlV4W1gpfB9tj8h+5gtFI0BVMktKhBBFXsoNPoFFJcpqfGg8MdiGEgCqnltnu2+pl+ANs9k/0DoVdbZQidSxWyUAtY34T8srqfIETJ68P4yt8Td7Y0f4f1lu9u+bTa1+k8eZsov'></script>
vendor: 1 bytes, line 1436
1436
1437<script id="sharing-js-js-after">
1438window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
1439window.WPCOM_sharing_popups[ "linkedin" ] = "menubar=1,resizable=1,width=580,height=450";
1440window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
1441window.WPCOM_sharing_popups[ "tumblr" ] = "menubar=1,resizable=1,width=450,height=450";
1442window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
1443window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
1444window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
1445window.WPCOM_sharing_popups[ "facebook" ] = "menubar=1,resizable=1,width=600,height=400";
1446//# sourceURL=sharing-js-js-after
1447</script>
vendor: 1 bytes, line 1447
1447
1448<script id="wp-emoji-settings" type="application/json">
1449{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63587"}}
1450</script>
vendor: 1 bytes, line 1450
1450
1451<script type="module">
1452/*! This file is auto-generated */
1453var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
1454//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
1455</script>
1455
1456	<iframe src='https://widgets.wp.com/likes/master.html?ver=20260924#ver=20260924' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
1457	<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
1458	
1458<script src="//stats.wp.com/w.js?69" defer></script>
1458 
1458<script type="text/javascript">
1459_tkq = window._tkq || [];
1460_stq = window._stq || [];
1461_tkq.push(['storeContext', {'blog_id':'113813661','blog_tz':'5','user_lang':'en','blog_lang':'en','user_id':'0'}]);
1462		// Prevent sending pageview tracking from WP-Admin pages.
1463		_stq.push(['view', {'blog':'113813661','v':'wpcom','tz':'5','user_id':'0','post':'289','subd':'cloudncode'}]);
1464		_stq.push(['extra', {'crypt':'UE5XaGUuOTlwaD85flAmcm1mcmZsaDhkV11YdWtpP0NsWnVkPS9sL0ViLndld3BuVT01Uj14Ti1yYTBnX3xuYUQ3Qz1oblZbVHV3MTZoVXFwTTVDK2wuPT1KWzJNV19iQk9bTmF0PUUlUlYzbW1nYyxHTnlKQiUsT0hKb2tYK09aZkN6LzJmYVtxMGpoLT9rJnJRJXg4UUR1VWN0dTBZTmxRSlN6JWx4cEtHLXErWnNjRzVselB1MkMsUEF5MkF5VVZ+Zm9tfmhoZG4wW2hOS05+VGRSUXRDcjhLdlg4cnRtZGhRVm9pOSYxMmw0L2tES0NFNU84JUlaJTc5UWFSMTE0T1lfMmJWLWpOVkpZLGZiWE9jYUEsS3luOWc4a2VtcmtGSnlpNjFDPWdJTFMvWlkxLF9jOFNSMUQvM2YmY0xuW1lDK19GN0Y9Q296RlhQampQeXR5SSxBaTlQaEpXU19nVm81X3pQQ10xL01ZZVk4aldGdC1xR1hZVD1DZGZtL1I1U1dCV3krWHYzakZtLmxKRVp8PS9fR3NyfCVNTGxbQ2kyRD8='}]);
1465_stq.push([ 'clickTrackerInit', '113813661', '289' ]);
1466</script>
1466
1467<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
1468<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="Asia/Kolkata" data-custom-props="{&quot;logged_in&quot;:&quot;0&quot;,&quot;wptheme&quot;:&quot;pub\/expound&quot;,&quot;wptheme_is_block&quot;:&quot;0&quot;}"   >
1469		
1469<script defer src="/wp-content/js/bilmur.min.js?i=19&amp;m=202639"></script>
1469 	
1470<script>
1471(function() {
1472	'use strict';
1473
1474	const fetches = {};
1475	const promises = {};
1476	const urls = {
1477		'wp-polyfill': '/wp-includes/js/dist/vendor/wp-polyfill.min.js?m=1786440387i&ver=3.15.0',
1478		'verbum': '/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/sun/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.js?m=1789668866i&minify=false&ver=1277503d9f869014c379'
1479	};
1480	const loaders = {
1481		'verbum': () => {
1482			fetchExternalScript('wp-polyfill');
1483			fetchExternalScript('verbum');
1484			promises['wp-polyfill'] = promises['wp-polyfill'] || loadWPScript('wp-polyfill');
1485			promises['verbum'] = promises['verbum'] || promises['wp-polyfill'].then( () => loadWPScript('verbum') );
1486			return promises['verbum'];
1487		},
1488		
1489	};
1490	const scriptExtras = {
1491		
1492	};
1493
1494	window.WP_Enqueue_Dynamic_Script = {
1495		loadScript: (handle) => {
1496			if (!loaders[handle]) {
1497				console.error('WP_Enqueue_Dynamic_Script: unregistered script `' + handle + '`.');
1498			}
1499			return loaders[handle]();
1500		}
1501	};
1502
1503	function fetchExternalScript(handle) {
1504		if (!urls[handle]) {
1505			return Promise.resolve();
1506		}
1507
1508		fetches[handle] = fetches[handle] || fetch(urls[handle], { mode: 'no-cors' });
1509		return fetches[handle];
1510	}
1511
1512	function runExtraScript(handle, type, index) {
1513		const id = 'wp-enqueue-dynamic-script:' + handle + ':' + type + ':' + (index + 1);
1514		const template = document.getElementById(id);
1515		if (!template) {
1516			return Promise.reject();
1517		}
1518
1519		const script = document.createElement( 'script' );
1520		script.innerHTML = template.innerHTML;
1521		document.body.appendChild( script );
1522		return Promise.resolve();
1523	}
1524
1525	function loadExternalScript(handle) {
1526		if (!urls[handle]) {
1527			return Promise.resolve();
1528		}
1529
1530		return fetches[handle].then(() => {
1531			return new Promise((resolve, reject) => {
1532				const script = document.createElement('script');
1533				script.onload = () => resolve();
1534				script.onerror = (e) => reject(e);
1535				script.src = urls[handle];
1536				document.body.appendChild(script);
1537			});
1538		});
1539	}
1540
1541	function loadExtra(handle, pos) {
1542		const count = (scriptExtras[handle] && scriptExtras[handle][pos]) || 0;
1543		let promise = Promise.resolve();
1544
1545		for (let i = 0; i < count; i++) {
1546			promise = promise.then(() => runExtraScript(handle, pos, i));
1547		}
1548
1549		return promise;
1550	}
1551
1552	function loadWPScript(handle) {
1553		// Core loads scripts in this order. See: https://github.com/WordPress/WordPress/blob/a59eb9d39c4fcba834b70c9e8dfd64feeec10ba6/wp-includes/class-wp-scripts.php#L428.
1554		return loadExtra(handle, 'translations')
1555			.then(() => loadExtra(handle, 'before'))
1556			.then(() => loadExternalScript(handle))
1557			.then(() => loadExtra(handle, 'after'));
1558	}
1559} )();
1560</script>
1560
1561
1562</body>
1563</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.