PageSourceSearch

https://www.ozkary.com/search?updated-max=2026-03-31T20:00:00-04:00&max-results=7

html ozkary.com collected 2026-10-03 19:25:19 UTC 144,668 bytes, 2,190 lines download raw bytes

1<!DOCTYPE html>
2<html dir='ltr' lang='en-US' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>
3<head>
4<link href='https://www.blogger.com/static/v1/widgets/4165133002-widget_css_bundle.css' rel='stylesheet' type='text/css'/>
5<meta charset='UTF-8'/>
6<meta content='width=device-width, initial-scale=1, maximum-scale=5' name='viewport'/>
7<link crossorigin='anonymous' href='https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css' rel='stylesheet'/>
8<link href='//maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css' rel='stylesheet'/>
9<link href='https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/styles/github-dark.min.css' rel='stylesheet'/>
10<meta content='no-referrer-when-downgrade' name='referrer'/>
11<meta content='max-image-preview:large' name='robots'/>
12<meta content='Óscar D. García' name='author'/>
13<meta content='Óscar D. García, ozkary, AIStorming, Data Engineering Process Fundamentals, ViCSA, Data Engineering, Enterprise Cloud Architecture, Agentic Systems, AI Assisted Discovery, Google Developer Group, Microsoft MVP, GDG Broward, Software Engineering, Python, SQL, Azure, GCP, BigQuery, Data Lakes' name='keywords'/>
14<meta content='Óscar D. García - ozkary' property='og:site_name'/>
15<meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>
16<meta content='blogger' name='generator'/>
17<link href='https://www.ozkary.com/favicon.ico' rel='icon' type='image/x-icon'/>
18<link href='https://www.ozkary.com/search?updated-max=2026-03-31T17:00:00-07:00&max-results=7' rel='canonical'/>
19<link rel="alternate" type="application/atom+xml" title="Ozkary -  Emerging Technologies - Atom" href="https://www.ozkary.com/feeds/posts/default" />
20<link rel="alternate" type="application/rss+xml" title="Ozkary -  Emerging Technologies - RSS" href="https://www.ozkary.com/feeds/posts/default?alt=rss" />
21<link rel="service.post" type="application/atom+xml" title="Ozkary -  Emerging Technologies - Atom" href="https://www.blogger.com/feeds/2250750882473644809/posts/default" />
22<link rel="me" href="https://www.blogger.com/profile/13427831719934915504" />
23<!--Can't find substitution for tag [blog.ieCssRetrofitLinks]-->
24<meta content='Software development, JavaScript, .NET, ASP.NET, Visual Studio, database, Azure, react, angular, NodeJS, chatbot, SQL, Data Engineering Fundamentals' name='description'/>
25<meta content='https://www.ozkary.com/search?updated-max=2026-03-31T17:00:00-07:00&max-results=7' property='og:url'/>
26<meta content='Ozkary -  Emerging Technologies' property='og:title'/>
27<meta content='Software development, JavaScript, .NET, ASP.NET, Visual Studio, database, Azure, react, angular, NodeJS, chatbot, SQL, Data Engineering Fundamentals' property='og:description'/>
28<meta content='https://www.ozkary.dev/assets/oscar-d-garcia-ozkary.png' property='og:image'/>
29<meta content='https://www.ozkary.dev/assets/oscar-d-garcia-ozkary.png' name='twitter:image'/>
30<meta content='website' property='og:type'/>
31<meta content='Óscar D. García - Cloud Data Architect, Author & Technologist' property='og:title'/>
32<meta content='https://ozkary.com/' property='og:url'/>
33<meta content='Principal Engineer, Cloud Architect, and Author of "Data Engineering Process Fundamentals". Chapter Lead for GDG Broward County. Sharing enterprise blueprints on AIStorming, agentic systems, and cloud data platforms.' property='og:description'/>
34<meta content='summary_large_image' name='twitter:card'/>
35<meta content='Senior Data Engineer, Cloud Architect, and Author of "Data Engineering Process Fundamentals". Chapter Lead for GDG Broward County. Sharing enterprise blueprints on AIStorming, agentic systems, and cloud data platforms.' name='twitter:description'/>
36<meta content='summary_large_image' name='twitter:card'/>
37<meta content='@ozkary' name='twitter:site'/>
38<meta content='@ozkary' name='twitter:creator'/>
39<meta content='Óscar D. García - Cloud Data Architect, Author & Technologist' name='twitter:title'/>
40<meta content='https://ozkary.com/' name='twitter:url'/>
41<script type='text/javascript'>
vendor: 374 bytes, lines 41-46
41
42        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
43        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
44        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
45        })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
46        ga('create', '
46UA-64364631-1
vendor: 31 bytes, lines 46-47
46', 'auto', 'blogger');
47        
47ga('blogger.send', 'pageview');
48      </script>
48
49<title>Ozkary -  Emerging Technologies</title>
50<style id='page-skin-1' type='text/css'><!--
51/* ======================================================
52JEKYLL / MINIMA-INSPIRED MODERN CLEAN THEME
53====================================================== */
54html {
55box-sizing: border-box;
56-webkit-text-size-adjust: 100%;
57}
58*, *:before, *:after {
59box-sizing: inherit;
60}
61body {
62background: #f8f9fa;
63color: #24292e;
64font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
65font-size: 16px;
66line-height: 1.6;
67margin: 0;
68padding: 0;
69}
70/* Container Structure */
71#pagepbt {
72max-width: 1140px;
73margin: 20px auto;
74padding: 0 15px;
75}
76#primarypbt {
77width: 70%;
78float: left;
79padding-right: 25px;
80}
81#secondarypbt {
82width: 30%;
83float: right;
84}
85/* Navigation Bar 1976d2*/
86.main-navigationpbt {
87background-color: #3d4144 !important;
88border-radius: 6px;
89margin-bottom: 20px;
90padding: 0.5rem 1rem;
91}
92.main-navigationpbt .nav-link {
93color: #ffffff !important;
94font-weight: 600;
95font-size: 14px;
96}
97.main-navigationpbt .nav-link:hover {
98opacity: 0.85;
99}
100/* Header & Bio */
101.site-headerpbt {
102background: #ffffff;
103border: 1px solid #e1e4e8;
104border-radius: 6px;
105padding: 20px;
106margin-bottom: 25px;
107box-shadow: 0 1px 3px rgba(0,0,0,0.04);
108}
109.site-titlenbt a {
110color: #111111;
111font-weight: 700;
112font-size: 32px;
113text-decoration: none;
114}
115.site-author {
116display: flex;
117align-items: center;
118gap: 15px;
119margin-top: 10px;
120}
121.author__avatar img {
122width: 60px;
123height: 60px;
124border-radius: 50%;
125}
126.site-descriptionnbt {
127font-size: 15px;
128color: #586069;
129font-weight: 400;
130margin: 0;
131}
132/* Cards & Content Blocks */
133.hentry, #secondarypbt .widget {
134background: #ffffff;
135border: 1px solid #e1e4e8 !important;
136border-radius: 6px !important;
137padding: 20px;
138margin-bottom: 25px;
139box-shadow: 0 1px 3px rgba(0,0,0,0.02);
140}
141header.entry-header h1.post-title a {
142color: #1a0dab;
143text-decoration: none;
144font-weight: 700;
145}
146header.entry-header h1.post-title a:hover {
147text-decoration: underline;
148}
149.entry-metapbt {
150font-size: 13px;
151color: #6a737d;
152margin-top: 5px;
153margin-bottom: 15px;
154}
155.entry-content {
156font-size: 16px;
157color: #24292e;
158}
159/* Typography Defaults */
160h1, h2, h3, h4, h5, h6 {
161color: #111111;
162font-weight: 600;
163margin-top: .75em;
164margin-bottom: 0.5em;
165}
166h2.title {
167font-size: 1.2rem;
168border-bottom: 2px solid #e1e4e8;
169padding-bottom: 5px;
170margin-bottom: 15px;
171}
172/* Responsive Images & Media Embeds */
173img {
174max-width: 100% !important;
175height: auto !important;
176}
177iframe {
178max-width: 100% !important;
179}
180/* Code Syntax & Snippets */
181pre {
182background: #0d1117;
183color: #c9d1d9;
184border-radius: 6px;
185padding: 16px;
186overflow: auto;
187font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo, monospace;
188font-size: 14px;
189position: relative;
190}
191/* Pro-Tip Custom Container */
192div.pro-tip {
193background-color: #f6f8fa;
194border: 1px solid #e1e4e8;
195border-left: 4px solid #0366d6;
196border-radius: 6px;
197padding: 12px 16px;
198margin: 15px 0;
199}
200div.pro-tip p {
201color: #0366d6 !important;
202font-weight: 700;
203margin: 0 !important;
204}
205/* Floating Clean Layout Clearing */
206.clear {
207clear: both;
208}
209/* Responsive Embed Wrapper */
210.responsive-video {
211position: relative;
212padding-bottom: 56.25%;
213height: 0;
214overflow: hidden;
215}
216.responsive-video iframe {
217position: absolute;
218top: 0;
219left: 0;
220width: 100%;
221height: 100%;
222}
223/* ======================================================
224MOBILE RESPONSIVE BREAKPOINTS
225====================================================== */
226@media screen and (max-width: 991px) {
227#primarypbt, #secondarypbt {
228width: 100% !important;
229float: none !important;
230padding-right: 0 !important;
231}
232}
233/* Responsive 16:9 Aspect Ratio for YouTube Embeds */
234.post-body iframe[src*="youtube.com"],
235.post-body iframe[src*="youtube-nocookie.com"],
236.responsive-video {
237width: 100% !important;
238aspect-ratio: 16 / 9 !important;
239height: auto !important;
240min-height: 240px; /* Fallback for older browsers */
241border-radius: 6px;
242margin: 15px 0;
243}
244/* Soft Charcoal Link Palette */
245a,
246header.entry-header h1.post-title a,
247.sidebar-sections a {
248color: #2c3e50 !important; /* Soft Charcoal */
249text-decoration: none;
250transition: color 0.2s ease-in-out;
251}
252a:hover,
253header.entry-header h1.post-title a:hover,
254.sidebar-sections a:hover {
255text-decoration: underline;
256}
257/* Update "Read Article" Buttons to match charcoal theme */
258.btn-outline-primary {
259color: #2c3e50 !important;
260border-color: #2c3e50 !important;
261}
262h1.post-title, h1 {
263font-size: 1.30rem !Important;
264}
265h2 {
266font-size: 1.25rem !Important;
267}
268h2.site-descriptionnbt {
269font-size: 1rem !Important;
270}
271h3, h4, h5, h6 {
272font-size: 1.15rem !Important;
273}
274.btn-outline-primary:hover {
275background-color: #2c3e50 !important;
276color: #ffffff !important;
277}
278html, body, #pagepbt, .site-contentpbt, #contentpbt {
279overflow: visible !important;
280overflow-x: visible !important;
281overflow-y: visible !important;
282transform: none !important; /* Disables transforms on wrappers that break position: fixed */
283}
284/* 2. Force MailMunch containers to anchor directly to the screen viewport */
285.mailmunch-popover,
286.mailmunch-slidein,
287.mailmunch-embedded,
288div[class*="mailmunch"],
289iframe[src*="mailmunch"] {
290position: fixed !important;
291z-index: 2147483647 !important; /* Maximum browser z-index depth */
292max-height: 100vh !important;
293box-sizing: border-box !important;
294}
295/* 3. If positioned at the bottom, ensure it isn't offset off-screen */
296.mailmunch-slidein {
297bottom: 0 !important;
298}
299@media screen and (max-width: 576px) {
300#pagepbt {
301padding: 0 10px;
302margin: 10px auto;
303}
304.site-author {
305flex-direction: column;
306text-align: center;
307}
308.hentry, #secondarypbt .widget {
309padding: 15px;
310}
311h1.site-titlenbt a {
312font-size: 1.30rem !important;
313}
314}
315
316--></style>
317<script>
vendor: 324 bytes, line 317
317(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','
317GTM-58XJDSH
vendor: 3 bytes, line 317
317');
317</script>
317
318<!-- End Google Tag Manager -->
319<link href='https://www.blogger.com/dyncss/2250750882473644809/authorization.css?zx=415be763-a341-4729-ab45-932f95a8db1d' media='none' onload='if(media!=&#39;all&#39;)media=&#39;all&#39;' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyncss/2250750882473644809/authorization.css?zx=415be763-a341-4729-ab45-932f95a8db1d' rel='stylesheet'/></noscript>
320<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
321<meta name='google-adsense-platform-domain' content='blogspot.com'/>
322
323<!-- data-ad-client=ca-pub-3622730893924152 -->
324
325</head>
326<body>
327<div id='pagepbt'>
328<nav class='navbar navbar-expand-lg navbar-dark main-navigationpbt' id='site-navigationpbt'>
329<a class='navbar-brand' href='https://www.ozkary.com/'>
330<img alt='ozkary' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgk1E7u4a1gReqqs69XwPlebQGzQ_FllmUo4MSHsAElIY1UIci5u3XxxL2ryqMLBxI65QmLb0gD2Gc5Yx8PuHx6r6bzhojBhP2lKvQ_FWFlk_7ksOTo9dIMk7vuFHGPdrgmFElE9tShyphenhyphent7kFDl-z99EhxTgS6lvkwt-Qk52hgzOMgxR7NJnl3b46UqdZWvy/s16000/ozkary-logo.png' style='width: 90px; border-radius: 4px; background: #fff; padding: 2px;'/>
331</a>
332<button aria-controls='global-menu' aria-expanded='false' aria-label='Toggle navigation' class='navbar-toggler' data-target='#global-menu' data-toggle='collapse' type='button'>
333<span class='navbar-toggler-icon'></span>
334</button>
335<div class='collapse navbar-collapse' id='global-menu'>
336<ul class='navbar-nav mr-auto'>
337<li class='nav-item'><a class='nav-link' href='https://github.com/ozkary' target='_blank'><i class='fa fa-github'></i> GitHub</a></li>
338<li class='nav-item'><a class='nav-link' href='https://twitter.com/ozkary' target='_blank'><i class='fa fa-twitter'></i> Twitter</a></li>
339<li class='nav-item'><a class='nav-link' href='https://www.youtube.com/@ozkary' target='_blank'><i class='fa fa-youtube-play'></i> YouTube</a></li>
340<li class='nav-item'><a class='nav-link' href='//dev.to/ozkary' target='_blank'><i class='fa fa-code'></i> DEV</a></li>
341<li class='nav-item'><a class='nav-link' href='https://www.ozkary.com/events'><i class='fa fa-bell'></i> Events</a></li>
342<li class='nav-item'><a class='nav-link' href='https://www.ozkary.com/newsletter'><i class='fa fa-envelope'></i> Newsletter</a></li>
343</ul>
344<ul class='navbar-nav ml-auto'>
345<li class='nav-item'><a class='nav-link' href='https://www.blogger.com/go/signin'><i class='fa fa-key'></i> Sign In</a></li>
346<li class='nav-item'><a class='nav-link' href='//www.ozkary.com/privacy'><i class='fa fa-lock'></i> Privacy</a></li>
347</ul>
348</div>
349</nav>
350<header class='site-headerpbt' role='banner'>
351<div class='headersec section' id='headersec'><div class='widget Header' data-version='1' id='Header1'>
352<div class='titlewrapper'>
353<h1 class='site-titlenbt'>
354<a href='https://www.ozkary.com/'>Ozkary -  Emerging Technologies</a>
355</h1>
356</div>
357<div class='site-author'>
358<div class='author__avatar'>
359<a href='/'>
360<img alt='Oscar D. Garcia - Ozkary' src='https://www.ozkary.dev/assets/oscar-d-garcia.png'/>
361</a>
362</div>
363<div class='site-bio'>
364<h2 class='site-descriptionnbt'>I am Oscar Garcia, Ozkary<sup>TM</sup>. I author this site, speak at conferences and events, contribute to OSS, mentor people. I use this blog to post ideas and experiences about software development, with the goal to both learn from and help the technology communities around the world.</h2>
365</div>
366</div>
367</div></div>
368</header>
369<div class='site-contentpbt' id='contentpbt'>
370<div class='content-areapbt' id='primarypbt'>
371<main class='site-mainpbt' id='mainpbt'>
372<div class='mainblogsec section' id='mainblogsec'><div class='widget Blog' data-version='1' id='Blog1'>
373<div class='blog-posts hfeed'>
374<article class='post hentry' itemprop='blogPost' itemscope='itemscope' itemtype='http://schema.org/BlogPosting'>
375<header class='entry-header'>
376<h1 class='post-title entry-title' itemprop='name'>
377<a href='https://www.ozkary.com/2026/02/ai-driven-architecture-smart-development-life-cycle-governance.html'>AI Driven App Architecture - Smart Development Life Cycle Governance</a>
378</h1>
379<div class='entry-metapbt'>
380<i class='fa fa-user'></i>&nbsp;<span itemprop='author' itemscope='itemscope' itemtype='https://schema.org/Person'><a href='https://www.ozkary.com/oscar-garcia-ozkary' itemprop='url'><span itemprop='name'>Oscar Garcia @ozkary</span></a></span>&nbsp;&nbsp;
381                      <i class='fa fa-calendar'></i>
3822/25/2026
383                      &nbsp;&nbsp;<i class='fa fa-clock-o'></i>
384<span class='post-read-time' data-post-id='3394714241175293616'>Loading read time...</span>
385</div>
386</header>
387<div class='post-body entry-content' id='post-body-3394714241175293616' itemprop='articleBody'>
388<div id='post-body-full-3394714241175293616' style='display: none;'><h1 id="overview">Overview</h1>
389<p>As development teams scale, maintaining architectural consistency becomes the biggest bottleneck. Documents are ignored, and linters only catch syntax errors, not design patterns.</p>
390<p>In this session, we will demonstrate how to transform AI from a passive coding assistant into an active Architectural Enfor
390cer. By embedding your &quot;unwritten rules&quot; directly into the repository configuration, you create a developer experience where the AI enforces your patterns in real-time.</p>
391<p>We will explore how this shifts the workflow: new developers are guided by the AI from day one, preventing architectural leakage before a pull request is ever opened.</p>
392<p><img alt="AI Driven App Architecture - Smart Development Life Cycle Governance" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-smart-development-life-cycle-governance.png" title="AI Driven App Architecture - Smart Development Life Cycle Governance"></p>
393<h2 id="-featured-open-source-projects">🚀 Featured Open Source Projects</h2>
394<p>Explore these curated resources to level up your engineering skills. If you find them helpful, a &#11088;&#65039; is much appreciated!</p>
395<h3 id="-data-engineering-https-github-com-ozkary-data-engineering-mta-turnstile-">🏗&#65039; <a href="https://github.com/ozkary/data-engineering-mta-turnstile">Data Engineering</a></h3>
396<blockquote>
397<p><strong>Focus:</strong> Real-world ETL &amp; MTA Turnstile Data<br><img alt="Maintained" loading="lazy" src="https://img.shields.io/badge/Maintained-Yes-green.svg"> <img alt="License" loading="lazy" src="https://img.shields.io/github/license/ozkary/data-engineering-mta-tur
397nstile"></p>
398</blockquote>
399<h3 id="-artificial-intelligence-https-github-com-ozkary-ai-engineering-">🤖 <a href="https://github.com/ozkary/ai-engineering">Artificial Intelligence</a></h3>
400<blockquote>
401<p><strong>Focus:</strong> LLM Patterns and Agentic Workflows<br><img alt="Status" loading="lazy" src="https://img.shields.io/badge/Status-Active_Development-blue.svg"> <img alt="Topic" loading="lazy" src="https://img.shields.io/badge/Focus-Generative_AI-orange"></p>
402</blockquote>
403<h3 id="-machine-learning-https-github-com-ozkary-machine-learning-engineering-">📉 <a href="https://github.com/ozkary/machine-learning-engineering">Machine Learning</a></h3>
404<blockquote>
405<p><strong>Focus:</strong> MLOps and Productionizing Models<br><img alt="Build" loading="lazy" src="https://img.shields.io/badge/Build-Passing-brightgreen.svg"> <img alt="Stage" loading="lazy" src="https://img.shields.io/badge/Stage-Production_Ready-blue"></p>
406</blockquote>
407<hr>
408<p>💡 <strong>Contribute:</strong> Found a bug or have a suggestion? Open an issue! and be part of the open source project.</p>
409<h2 id="youtube-video">YouTube Video</h2>
410<iframe width="560" height="315" src="https://www.youtube.com/embed/wvhb9B3DeMY?si=gRHAES40_s1HdMkX" title="AI Driven App Architecture - Smart Development Life Cycle Governance" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
411
412<blockquote>
413<p>👍 Subscribe to the channel to get notify on new events!</p>
414</blockquote>
415<h3 id="video-agenda">Video Agenda</h3>
416<p><strong>The Problem: Architectural Drift</strong></p>
417<p>Why strict rules (Controller-View, Pascal/camelCase) degrade over time and how AI can fix it.</p>
418<p><strong>The Intelligence Engine</strong></p>
419<p>Breakdown of the core components: Global Rules, Contextual Guardrails, Agent Tools, and Directory Structure.</p>
420<p><strong>Configuration: Global Governance</strong></p>
421<p>Setting up global &quot;system prompts&quot; for the repository to enforce tech stack and naming conventions.</p>
422<p><strong>Configuration: Contextual Guardrails</strong></p>
423<p>Creating &quot;firewalls&quot; for specific folders (e.g., preventing logic in views, preventing API calls in Controllers).</p>
424<p><strong>Configuration: The Tooling</strong></p>
425<p>Building custom Slash Commands (/new-module) to automate &quot;Vertical Slice&quot; scaffolding.</p>
426<p><strong>Configuration: The Auditor Agent</strong></p>
427<p>Implementing a specialized &quot;Gatekeeper&quot; persona that scans imports to ensure strict layer separation.</p>
428<p><strong>Agent Mapping</strong></p>
429<p>A conceptual framework comparing repository configuration to autonomous agent architecture.</p>
430<p><strong>💡 Why Attend?</strong></p>
431<ul>
432<li>Stop writing boilerplate: Learn to automate complex folder structures with one command.</li>
433<li>Reduce PR Reviews: Shift governance &quot;left&quot; by having the AI catch architectural errors instantly.</li>
434<li>Interactive Demo: See the .github configuration in action on a real codebase.</li>
435<li>Takeaway Code: Leave with the copy-paste markdown templates to implement this in your own repo tomorrow.</li>
436</ul>
437<p><strong>Target Audience</strong></p>
438<ul>
439<li>Tech Leads &amp; Architects who need to enforce standards across scaling teams.</li>
440<li>Developers who are tired of correcting the same patterns in code reviews.</li>
441<li>DevOps Engineers interested in &quot;Governance as Code.&quot;</li>
442<li>Leadership teams that are trying to raise standards and productivity in their organizations.</li>
443</ul>
444<h2 id="presentation">Presentation</h2>
445<h3 id="setting-the-stage">SETTING THE STAGE</h3>
446<p><strong>The Context</strong></p>
447<ul>
448<li>We enforce a strict pattern using the ViCSA architecture</li>
449<li>PascalCase for UI Components.</li>
450<li>camelCase for Logic &amp; Services.</li>
451<li>Separation of Concerns (SoC)  is non-negotiable.</li>
452</ul>
453<p><strong>The Problem</strong></p>
454<ul>
455<li>Architectural Drift: Patterns degrade over time.</li>
456<li>Passive Docs: Wiki pages are ignored.</li>
457<li>Linter Limits: Linters catch syntax, not architecture.</li>
458<li>Solution: Active Governance via AI.</li>
459</ul>
460<h3 id="the-intelligence-engine">THE INTELLIGENCE ENGINE</h3>
461<p><strong>Core AI Policies</strong></p>
462<ul>
463<li>Centralized Config: Rules live in the repo, not the user&#39;s IDE.</li>
464<li>Global Rules: Applied to every interaction (System Prompt).</li>
465<li>Contextual Rules: Triggered only when specific files are opened.</li>
466<li>Agent Tools: Custom commands to scaffold new components, controllers or services.</li>
467</ul>
468<p><img alt="AI Driven App Architecture - Smart Development Life Cycle Governance - Project Structure" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-project-structure.png"></p>
469<h3 id="configuration-global-governance">CONFIGURATION: GLOBAL GOVERNANCE</h3>
470<p><strong>Global Instructions</strong></p>
471<p><strong>File:</strong> <code>.github/copilot-instructions.md</code></p>
472<p>This acts as the System Prompt for the entire repository. It is silently added to every interaction.</p>
473<ul>
474<li>Tech Stack: TS, Tailwind, Hooks.</li>
475<li>Naming: Pascal vs camelCase.</li>
476<li>Flow: <code>View &#8594; Controller &#8594; Service -&gt; API</code>.</li>
477</ul>
478<p><img alt="AI Driven App Architecture - Smart Development Life Cycle Governance - Global Governance" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-global-governance.png"></p>
479<h3 id="dev-experience-the-silent-enforcer">DEV EXPERIENCE: THE SILENT ENFORCER</h3>
480<p><strong>Without Config</strong></p>
481<p>A developer asks: </p>
482<p><code>How do I create a new service?</code></p>
483<ul>
484<li>AI suggests a generic Class-based service.</li>
485<li>Suggests creating a utils.js file.</li>
486<li>Ignores project folder structure.</li>
487</ul>
488<p><strong>With Config</strong></p>
489<p>A developer asks: 
490<code>How do I create a new service?&quot;</code></p>
491<ul>
492<li>AI reads the Governance.</li>
493<li>Response: <code>Create src/services/userAuth/index.ts using a functional export, as per project standards.</code></li>
494</ul>
495<h3 id="configuration-contextual-guardrails">CONFIGURATION: CONTEXTUAL GUARDRAILS</h3>
496<p><strong>View Layer Rules</strong></p>
497<p><strong>File:</strong> <code>.github/instructions/controller-layer.md</code></p>
498<p><strong>Trigger:</strong> Opening any <code>
498**/*.tsx file</code>.</p>
499<ul>
500<li>&quot;You are a View.&quot;</li>
501<li>&quot;No Logic allowed.&quot;</li>
502<li>&quot;No direct API calls.&quot;</li>
503</ul>
504<p><strong>Controller Layer Rules</strong></p>
505<p><strong>File:</strong> <code>.github/instructions/view-layer.md</code></p>
506<p><strong>Trigger:</strong> Opening any <code>**/controller.ts</code> file.</p>
507<ul>
508<li>&quot;You are a Controller.&quot;</li>
509<li>&quot;Use Services, NOT Fetch.&quot;</li>
510<li>&quot;Manage State here.&quot;</li>
511</ul>
512<h3 id="dev-experience-real-time-intervention">DEV EXPERIENCE: REAL-TIME INTERVENTION</h3>
513<p><strong>The Scenario</strong></p>
514<ul>
515<li>A developer tries to write fetch() inside a UI Component (index.tsx).</li>
516<li>They ask Copilot: &quot;Write a fetch call here for me.&quot;</li>
517</ul>
518<p><strong>The Intervention</strong></p>
519<p><strong>Ghost Text:</strong> Copilot refuses to autocomplete the network call.</p>
520<p><strong>Chat Reply:</strong></p>
521<p> <code>I cannot. This is a View file. Please move this logic to the sibling Controller (index.ts) and import it.</code> </p>
522<h3 id="configuration-the-tooling">CONFIGURATION: THE TOOLING</h3>
523<p><strong>Prompt Library</strong></p>
524<p><strong>File:</strong> <code>.github/prompts/new-module.md</code></p>
525<p>These act as Agent Tools or &quot;Slash Commands&quot;.</p>
526<ul>
527<li>Goal: Automate the &quot;Vertical Slice&quot;.</li>
528<li>Benefit: Complex scaffolding logic is stored in the repo, not in the developer&#39;s head.</li>
529<li>Usage: <code>/new-module</code></li>
530</ul>
531<pre><code># Prompt Library (The Scaffolder)
532File: `.github/prompts/new-component.md`
533Goal: Automate the creation <span class="hljs-keyword">of</span> a standalone UI Component <span class="hljs-keyword">with</span> optional Service/API layers.
534
535# Create New Component
536I need to generate a new component following our **Folder-<span class="hljs-keyword">as</span>-Namespace** pattern.
537**Command:** `/new-component:{{componentName}} {{args}}`
538
539Please generate the <span class="hljs-keyword">code</span> blocks for the layers requested <span class="hljs-keyword">in</span> the arguments (service, api). 
540*Note: Logic folders must be camelCase. UI folders must be PascalCase.*
541
542---
543
544### Component Layer (Required)
545**Folder:** `src/components/{{componentName (PascalCase)}}/`
546- **File:** `controller.ts` (Controller): Logic and State only.
547- **File:** `index.tsx` (View): Pure UI. Imports Controller.
548---
549
550
551### Service Layer (Optional)
552*Condition: Generate only <span class="hljs-keyword">if</span> <span class="hljs-string">'service'</span> is present <span class="hljs-keyword">in</span> {{args}}.*
553
554**File:** `src/services/{{componentName (camelCase)}}/index.ts`
555- **Role:** Business logic and data transformation.
556- **Code:** Import the API (<span class="hljs-keyword">if</span> requested). Export a service object or functional exports.
557
558---
559
560### API Layer (Optional)
561*Condition: Generate only <span class="hljs-keyword">if</span> <span class="hljs-string">'api'</span> is present <span class="hljs-keyword">in</span> {{args}}.*
562
563**File:** `src/apis/{{componentName (camelCase)}}/index.ts`
564- **Role:** Define specific endpoints.
565- **Code:** Import `coreClient` <span class="hljs-keyword">from</span> `src/apis/index.ts`. Export async functions <span class="hljs-keyword">with</span> typed responses.
566
567---
568
569### Style Guidelines
570- **Typing:** Use TypeScript interfaces for all Props and Data models.
571- **Separation:** Logic stays <span class="hljs-keyword">in</span> `controller.ts`, JSX stays <span class="hljs-keyword">in</span> `index.tsx`.
572- **Naming:** Components use PascalCase; Services/APIs use camelCase.
573</code></pre><h3 id="dev-experience-the-scaffolding">DEV EXPERIENCE: THE SCAFFOLDING</h3>
574<p><strong>The Command</strong></p>
575<p>Starting a new feature called &quot;Sales Dashboard&quot;.</p>
576<p><strong>Action:</strong></p>
577<p><code>/new-module featureName:Sales Dashboard</code></p>
578<p><strong>The Execution</strong></p>
579<ul>
580<li>Analyzes the request.</li>
581<li>Applies <code>PascalCase</code> to Containers/Components folders.</li>
582<li>Applies <code>camelCase</code> to api/service folders.</li>
583<li>Generates the <code>Controller-View</code> pair instantly.</li>
584</ul>
585<h3 id="the-result-generated-architecture">THE RESULT: GENERATED ARCHITECTURE</h3>
586<p><strong>The Results</strong></p>
587<ul>
588<li>Layers generated instantly.</li>
589<li>Correct naming conventions applied.</li>
590<li>Zero manual boilerplate.</li>
591</ul>
592<p><img alt="AI Driven App Architecture - Smart Development Life Cycle Governance - Project Structure" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-project-structure.png"></p>
593<h3 id="configuration-the-auditor-agent">CONFIGURATION: THE AUDITOR AGENT</h3>
594<p><strong>Specialized Persona</strong></p>
595<p><strong>File:</strong> <code>.github/agents/arch-auditor.md</code></p>
596<p>This creates a named Agent that acts as a Gatekeeper. It doesn&#39;t write features; it verifies them.</p>
597<ul>
598<li>Role: Architecture Enforcer.</li>
599<li>Task: Scans imports to ensure strict layer separation.</li>
600<li>Rule: &quot;Views never talk to APIs.&quot;</li>
601</ul>
602<pre><code># Custom AI Agent (The Reviewer)
603Agent ID: `@vicsa-auditor`
604
605Context: A bot that ensures the chain <span class="hljs-keyword">of</span> command is respected using the ViCSA architecture (View Controller Service API)
606
607## Primary Objective
608name: Architecture Auditor
609description: Verifies strict separation <span class="hljs-keyword">of</span> Controller, Service, and View layers.
610tools: [<span class="hljs-keyword">code</span>-search]
611
612---
613## Role
614You ensure the integrity <span class="hljs-keyword">of</span> the data flow: View -&gt; Controller -&gt; Service -&gt; API.
615
616## Audit Logic
617When asked to <span class="hljs-string">"Audit this feature"</span>:
618
619<span class="hljs-number">1.</span> **Check the View (.tsx):** - FAIL <span class="hljs-keyword">if</span> it imports `src/services`.
620   - FAIL <span class="hljs-keyword">if</span> it imports `src/apis`.
621   - PASS only <span class="hljs-keyword">if</span> it imports `./index`.
622
623<span class="hljs-number">2.</span> **Check the Controller (.ts):**
624   - FAIL <span class="hljs-keyword">if</span> it uses `fetch` or `axios`.
625   - PASS only <span class="hljs-keyword">if</span> it delegates to `src/services`.
626
627<span class="hljs-number">3.</span> **Check the Service:**
628   - FAIL <span class="hljs-keyword">if</span> it defines its own URL logic.
629   - PASS only <span class="hljs-keyword">if</span> it imports `src/apis/index.ts`.
630</code></pre><h3 id="dev-experience-the-code-review">DEV EXPERIENCE: THE CODE REVIEW</h3>
631<p><strong>The Interaction</strong></p>
632<p>Before raising a pull request, the developer invokes the auditor.</p>
633<p><strong>Prompt:</strong></p>
634<p><code>@vicsa-auditor check this component for violations.</code></p>
635<p><strong>Response:</strong></p>
636<p> <code>&#9989; PASS: SalesDashboard/index.tsx imports only from its sibling controller. No direct API calls found.</code></p>
637<p><img alt="AI Driven App Architecture - Smart Development Life Cycle Governance - Review Process" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-review-process.png"></p>
638<h3 id="the-autonomy-advantage">THE AUTONOMY ADVANTAGE</h3>
639<p>AI enforces the ViCSA architecture through continuous observation and autonomous execution.</p>
640<ul>
641<li><strong>Perception</strong>: Continuously observes the active workspace, file paths (e.g., src/components/), and context to understand the developer&#39;s structural intent.</li>
642<li><strong>Reasoning</strong>: Evaluates the perceived context against the repository&#39;s .github Guardrails, determining if a View is bypassing a Controller or violating Separation of Concerns, SoC.</li>
643<li><strong>Action</strong>: Executes autonomous scaffolding, enforces strict ViCSA governance, provides recommended fixes feedback.</li>
644</ul>
645<h3 id="summary-agent-mapping">SUMMARY &amp; AGENT MAPPING</h3>
646<p>Embedding governance directly into the repository transforms the development lifecycle. It replaces passive wiki pages with active, real-time enforcement, ensuring that every AI suggestion aligns with architectural standards. This eliminates &quot;drift&quot;, accelerates onboarding, and turns Copilot into a domain-expert partner.</p>
647<table>
648<thead>
649<tr>
650<th>Agent Component</th>
651<th>GitHub Implementation</th>
652</tr>
653</thead>
654<tbody>
655<tr>
656<td>System Prompt</td>
657<td>Global Instructions (copilot-instructions.md)</td>
658</tr>
659<tr>
660<td>Context / RAG</td>
661<td>Modular Instructions (instructions/*.md)</td>
662</tr>
663<tr>
664<td>Tools / Functions</td>
665<td>Prompt Library (prompts/*.md)</td>
666</tr>
667<tr>
668<td>Human Prompt</td>
669<td>Chat Window</td>
670</tr>
671<tr>
672<td>Persona</td>
673<td>Agent Personas (i.e. agents/arch-auditor.md)</td>
674</tr>
675</tbody>
676</table>
677<blockquote>
678<p>RAG: Retrieval augmented generation</p>
679</blockquote>
680<h3 id="-let-s-connect-build-together">🌟 Let&#39;s Connect &amp; Build Together</h3>
681<p>Thanks for reading! 😊 If you enjoyed these resources, let&#39;s stay in touch! I share deep-dives into AI/ML patterns and host community events here:</p>
682<ul>
683<li><strong><a href="https://gdg.community.dev/gdg-broward-county-fl/">GDG Broward</a></strong>: Join our local dev community for meetups and workshops.</li>
684<li><strong><a href="https://globalai.community/chapters/jacksonville/">Global AI Events</a></strong>: Join Global AI Events.</li>
685<li><strong><a href="https://www.linkedin.com/in/oscardgarcia">LinkedIn</a></strong>: Let&#39;s connect professionally! I share insights on engineering.</li>
686<li><strong><a href="https://github.com/ozkary">GitHub</a></strong>: Follow my open-source journey and star the repos you find useful.</li>
687<li><strong><a href="https://www.youtube.com/@ozkary">YouTube</a></strong>: Watch step-by-step tutorials on the projects listed above.</li>
688<li><strong><a href="https://bsky.app/profile/ozkary.bsky.social">BlueSky</a></strong> / <strong><a href="https://x.com/ozkary">X / Twitter</a></strong>: Daily tech updates and quick engineering tips.</li>
689</ul>
690</div>
691<div class='post-teaser' style='display: flex; gap: 15px; flex-wrap: wrap; margin-bottom: 15px;'>
692<div class='teaser-thumbnail' style='max-width: 200px; flex: 1 1 150px;'>
693<a href='https://www.ozkary.com/2026/02/ai-driven-architecture-smart-development-life-cycle-governance.html'>
694<img alt='Post Thumbnail' itemprop='image' loading='lazy' src='https://www.ozkary.dev/assets/2026/ozkary-ai-driven-architecture-smart-development-life-cycle-governance.png' style='width: 100%; height: auto; border-radius: 6px; object-fit: cover;'/>
695</a>
696</div>
697<div class='teaser-snippet' style='flex: 2 1 250px;'>
698<p style='margin-bottom: 10px; color: #4a5568;'>
699Overview  As development teams scale, maintaining architectural consistency becomes the biggest bottleneck. Documents are ignored, and linte...
700</p>
701<a class='btn btn-sm btn-outline-primary' href='https://www.ozkary.com/2026/02/ai-driven-architecture-smart-development-life-cycle-governance.html'>Read Article &rarr;</a>
702</div>
703</div>
704<div class='clear'></div>
705</div>
706</article>
707<article class='post hentry' itemprop='blogPost' itemscope='itemscope' itemtype='http://schema.org/BlogPosting'>
708<header class='entry-header'>
709<h1 class='post-title entry-title' itemprop='name'>
710<a href='https://www.ozkary.com/2026/01/the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment.html'>The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment</a>
711</h1>
712<div class='entry-metapbt'>
713<i class='fa fa-user'></i>&nbsp;<span itemprop='author' itemscope='itemscope' itemtype='https://schema.org/Person'><a href='https://www.ozkary.com/oscar-garcia-ozkary' itemprop='url'><span itemprop='name'>Oscar Garcia @ozkary</span></a></span>&nbsp;&nbsp;
714                      <i class='fa fa-calendar'></i>
7151/21/2026
716                      &nbsp;&nbsp;<i class='fa fa-clock-o'></i>
717<span class='post-read-time' data-post-id='1775744515635833471'>Loading read time...</span>
718</div>
719</header>
720<div class='post-body entry-content' id='post-body-1775744515635833471' itemprop='articleBody'>
721<div id='post-body-full-1775744515635833471' style='display: none;'><h1 id="overview">Overview</h1>
722<p>In the modern data landscape, the wall between &quot;where data lives&quot; and &quot;how we get insights&quot; is crumbling. This session focuses on the Cognitive Data Lakehouse. A paradigm shift that allows developers to treat a fragmented data lake as a unified, high-performance warehouse.</p>
723<p>We will explore how to move beyond brittle ETL pipelines using Zero-ETL architecture in the cloud. The core of our discussion 
723will center on using integrated AI capabilities and semantic modeling to solve the &quot;Metadata Mess&quot; inherent in global manufacturing feeds without moving a single byte of data. From raw telemetry in object storage to semantic intelligence via large language models, we&#8217;ll show you the real-world application of AI in modern data engineering.</p>
724<p><img alt="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment-sm.png" title="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment"></p>
725<h2 id="-featured-open-source-projects">🚀 Featured Open Source Projects</h2>
726<p>Explore these curated resources to level up your engineering skills. If you find them helpful, a &#11088;&#65039; is much appreciated!</p>
727<h3 id="-data-engineering-https-github-com-ozkary-data-engineering-mta-tur
727nstile-">🏗&#65039; <a href="https://github.com/ozkary/data-engineering-mta-turnstile">Data Engineering</a></h3>
728<blockquote>
729<p><strong>Focus:</strong> Real-world ETL &amp; MTA Turnstile Data<br><img alt="Maintained" loading="lazy" src="https://img.shields.io/badge/Maintained-Yes-green.svg"> <img alt="License" loading="lazy" src="https://img.shields.io/github/license/ozkary/data-engineering-mta-turnstile"></p>
730</blockquote>
731<h3 id="-artificial-intelligence-https-github-com-ozkary-ai-engineering-">🤖 <a href="https://github.com/ozkary/ai-engineering">Artificial Intelligence</a></h3>
732<blockquote>
733<p><strong>Focus:</strong> LLM Patterns and Agentic Workflows<br><img alt="Status" loading="lazy" src="https://img.shields.io/badge/Status-Active_Development-blue.svg"> <img alt="Topic" loading="lazy" src="https://img.shields.io/badge/Focus-Generative_AI-orange"></p>
734</blockquote>
735<h3 id="-machine-learning-https-github-com-ozkary-machine-learning-engineering-">📉 <a href="https://github.com/ozkary/machine-learning-engineering">Machine Learning</a></h3>
736<blockquote>
737<p><strong>Focus:</strong> MLOps and Productionizing Models<br><img alt="Build" loading="lazy" src="https://img.shields.io/badge/Build-Passing-brightgreen.svg"> <img alt="Stage" loading="lazy" src="https://img.shields.io/badge/Stage-Production_Ready-blue"></p>
738</blockquote>
739<hr>
740<p>💡 <strong>Contribute:</strong> Found a bug or have a suggestion? Open an issue! and be part of the open source project.</p>
741<h2 id="youtube-video">YouTube Video</h2>
742<iframe width="560" height="315" src="https://www.youtube.com/embed/nfJl-4BxqyY?si=mHyV5N547HqZ0rJx" title="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
743
744<h3 id="video-agenda">Video Agenda</h3>
745<p><strong>Phase 1: Foundations &amp; The Zero-ETL Strategy</strong></p>
746<p>We kick off with the infrastructure layer. We&#39;ll discuss the design of cross-region telemetry tables and how modern cloud engines allow us to query raw files in object storage with the performance of a native table. We&#8217;ll establish why &quot;0x data movement&quot; is the goal for modern scalability.</p>
747<p><strong>Phase 2: Confronting the Metadata Mess</strong></p>
748<p>Schema drift and inconsistent naming across global regions are the enemies of unified analytics. We will look at why traditional manual mapping fails and how we can use AI inference to bridge these gaps and standardize naming conventions automatically.</p>
749<p><strong>Phase 3: AI-Driven Unification &amp; Semantic Modeling</strong></p>
750<p>The &quot;Cognitive&quot; part of the Lakehouse. We&#8217;ll dive into the technical implementation of registering AI models directly within your data warehouse environment. You&#39;ll see how to create an abstraction layer that uses AI to normalize data on the fly, creating a robust semantic model.</p>
751<p><strong>Phase 4: Scaling to a Global Feed</strong></p>
752<p>Finally, we&#8217;ll demonstrate the DevOps workflow for integrating a new international factory feed into a global telemetry view. We&#39;ll show how to maintain a &quot;Single Source of Intelligence&quot; that BI tools and analysts can consume without needing to know the complexities of the underlying lake.</p>
753<p><strong>💡 Why Attend?</strong></p>
754<ul>
755<li>Master Modern Architecture: Learn the &quot;Abstraction Layer&quot; design pattern that is replacing traditional, slow ETL/ELT processes.</li>
756<li>Hands-on AI for Data Ops: See exactly how to use AI and semantic modeling within SQL-based workflows to automate data cleaning and schema mapping.</li>
757<li>Scale Without Pain: Discover how to manage global data sources (multi-region, multi-format) through a single governing layer.</li>
758<li>Developer Networking: Connect with other data architects, engineering leaders, and professionals solving similar scale and complexity challenges.</li>
759</ul>
760<p><strong>Target Audience:</strong> Data Engineers, Analytics Architects, Cloud Developers, and anyone interested in the intersection of Big Data and Generative AI.</p>
761<h2 id="presentation">Presentation</h2>
762<h3 id="phase-1-the-zero-etl-strategy">Phase 1: The Zero-ETL Strategy</h3>
763<h4 id="infrastructure-data-stays-local">INFRASTRUCTURE: DATA STAYS LOCAL</h4>
764<p><strong>Architecting for Scale</strong></p>
765<ul>
766<li>Storage Decoupling: Raw files remain in the Data Lake, eliminating replication overhead.</li>
767<li>Virtual Access: Data Warehouse external tables allow immediate querying of CSV, Parquet, and JSON.</li>
768<li>Minimal Latency: No waiting for ingest pipelines; analysis starts upon file arrival.</li>
769</ul>
770<p><img alt="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment -  Medallion Architecture Design Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-raw-zone.png" title="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment - Data Stays Local "></p>
771<h4 id="unmatched-storage-efficiency">UNMATCHED STORAGE EFFICIENCY</h4>
772<p><strong>Zero Data Replication</strong></p>
773<ul>
774<li>Traditional ETL requires moving data across multiple tiers. Our architecture ensures a single source of truth with zero data movement between GCS and BigQuery compute.</li>
775<li>This is similar to the Bronze Zone in a Medallion Architecture.</li>
776</ul>
777<p><img alt="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment -  Medallion Architecture Design Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-bronze-zone.png" title="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment - Storage Efficiency"></p>
778<h3 id="phase-2-the-metadata-mess">Phase 2: The Metadata Mess</h3>
779<h4 id="challenges-of-unification">CHALLENGES OF UNIFICATION</h4>
780<p><strong>Schema Friction</strong></p>
781<ul>
782<li>Feeds arrive with inconsistent headers (e.g., &#39;Device Number&#39; vs &#39;deviceNo&#39;). Manual aliasing is fragile and slow.</li>
783</ul>
784<p><strong>Entity Drift</strong></p>
785<ul>
786<li>Names and IDs vary across systems, preventing standard joins from matching records effectively.</li>
787</ul>
788<p><strong>Type Mismatches</strong></p>
789<ul>
790<li>Varying data types for the same concept (Integer vs String) crash standard SQL aggregation views.</li>
791</ul>
792<h3 id="phase-3-the-ai-solution">Phase 3: The AI Solution</h3>
793<h4 id="bigquery-studio-the-ai-interface">BIGQUERY STUDIO: THE AI INTERFACE</h4>
794<p><strong>Remote AI Registration</strong></p>
795<ul>
796<li>Register Gemini Pro directly inside BigQuery to enable cognitive functions within your SQL workspace.</li>
797</ul>
798<pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">MODEL</span> <span class="hljs-string">`gemini_remote`</span>
799REMOTE <span class="hljs-keyword">WITH</span> <span class="hljs-keyword">CONNECTION</span> <span class="hljs-string">`bq_connection`</span>
800OPTIONS(endpoint = <span class="hljs-string">'gemini-1.5-pro'</span>);
801</code></pre>
802<p><strong>Automated Inference</strong></p>
803<ul>
804<li>AI &quot;reads&quot; information schemas to infer mapping logic, moving you from Code Author to Logic Approver.</li>
805</ul>
806<pre><code class="lang-sql"><span class="hljs-keyword">SELECT</span> ml_generate_text_result
807<span class="hljs-keyword">FROM</span> ML.GENERATE_TEXT(
808  <span class="hljs-keyword">MODEL</span> <span class="hljs-string">`gemini_remote`</span>,
809  (<span class="hljs-keyword">SELECT</span> <span class="hljs-string">"Compare Source A and B schemas. Write a SQL view to unify them."</span> <span class="hljs-keyword">AS</span> <span class="hljs-keyword">prompt</span>)
810);
811</code></pre>
812<h4 id="ai-assisted-schema-discovery">AI-ASSISTED SCHEMA DISCOVERY</h4>
813<p><strong>Prompting for Base Tables</strong></p>
814<ul>
815<li>Using AI to generate the DDL for external tables by pointing to compressed feeds in the lake (USA &amp; MEX factories).</li>
816</ul>
817<pre><code class="lang-sql"><span class="hljs-keyword">SELECT</span> ml_generate_text_result
818<span class="hljs-keyword">FROM</span> ML.GENERATE_TEXT(
819  <span class="hljs-keyword">MODEL</span> <span class="hljs-string">`gemini_remote`</span>,
820  (<span class="hljs-keyword">SELECT</span> <span class="hljs-string">"Create External Tables as smart_factory.us_telemetry with path 'gs://factory-dl/us/dev-540/telemetry-*.csv.gz' '. Include option CSV, GZIP compression and skip 1 row. Infer and add the schema using lower case"</span> <span class="hljs-keyword">AS</span> <span class="hljs-keyword">prompt</span>));
821
822<span class="hljs-keyword">SELECT</span> ml_generate_text_result
823<span class="hljs-keyword">FROM</span> ML.GENERATE_TEXT(
824  <span class="hljs-keyword">MODEL</span> <span class="hljs-string">`gemini_remote`</span>,
825  (<span class="hljs-keyword">SELECT</span> <span class="hljs-string">"Create External Tables as smart_factory.mx_telemetry with path 'gs://factory-dl/mx/dev-940/telemetry-*.csv.gz' '. Include option CSV, GZIP compression and skip 1 row. Use schema device_number STRING, bay_id INT64, factory 
825STRING, created STRING"</span> <span class="hljs-keyword">AS</span> <span class="hljs-keyword">prompt</span>));
826</code></pre>
827<p><strong>Generated BigLake DDL</strong></p>
828<pre><code class="lang-sql"><span class="hljs-comment">-- USA Factory Feed</span>
829<span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">OR</span> <span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">EXTERNAL</span> <span class="hljs-keyword">TABLE</span> <span class="hljs-string">`smart_factory.us_telemetry`</span> (
830  device_number <span class="hljs-keyword">STRING</span>,
831  bay_id INT64,
832  factory <span class="hljs-keyword">STRING</span>,
833  created <span class="hljs-keyword">STRING</span>
834)
835OPTIONS (
836  <span class="hljs-keyword">format</span> = <span class="hljs-string">'CSV'</span>,
837  uris = [<span class="hljs-string">'gs://factory-dl/us/dev-540/telemetry*.csv.gz'</span>],
838  skip_leading_rows = <span class="hljs-number">1</span>,
839  compression = <span class="hljs-string">'GZIP'</span>
840);
841
842<span class="hljs-comment">-- MEX Factory Feed</span>
843<span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">OR</span> <span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">EXTERNAL</span> <span class="hljs-keyword">TABLE</span> <span class="hljs-string">`smart_factory.mx_telemetry`</span> (
844  device_number <span class="hljs-keyword">STRING</span>,
845  bay_id INT64,
846  factory <span class="hljs-keyword">STRING</span>,
847  created <span class="hljs-keyword">STRING</span>
848)
849OPTIONS (
850  <span class="hljs-keyword">format</span> = <span class="hljs-string">'CSV'</span>,
851  uris = [<span class="hljs-string">'gs://factory-dl/mx/dev-940/telemetry*.csv.gz'</span>],
852  skip_leading_rows = <span class="hljs-number">1</span>,
853  compression = <span class="hljs-string">'GZIP'</span>
854);
855</code></pre>
856<h4 id="ai-abstraction-the-view-layer">AI-ABSTRACTION: THE VIEW LAYER</h4>
857<p><strong>Generating the Interface</strong></p>
858<ul>
859<li>AI creates a clean abstraction view for each external table, decoupling raw storage from the analytics model.<pre><code class="lang-sql"><span class="hljs-comment">-- AI Instruction</span>
860"<span class="hljs-keyword">Create</span> a <span class="hljs-keyword">view</span> named 
861smart_factory.vw_us_telemetry 
862selecting all <span class="hljs-keyword">columns</span> <span class="hljs-keyword">from</span> the
863usa_telemetry table. <span class="hljs-keyword">Safe</span> <span class="hljs-keyword">cast</span> the created <span class="hljs-keyword">column</span> <span class="hljs-keyword">as</span> datetime.<span class="hljs-string">"</span>
864</code></pre>
865</li>
866</ul>
867<p><strong>Abstraction Layer DDL</strong></p>
868<pre><code class="lang-sql"><span class="hljs-comment">-- Semantic Abstraction Layer</span>
869<span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">OR</span> <span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">VIEW</span> <span class="hljs-string">`smart_factory.vw_us_telemetry`</span> <span class="hljs-keyword">AS</span>
870<span class="hljs-keyword">SELECT</span> 
871  device_number,
872  bay_id,
873  factory,
874  SAFE_CAST(created <span class="hljs-keyword">as</span> DATETIME) <span class="hljs-keyword">AS</span> created
875<span class="hljs-keyword">FROM</span> <span class="hljs-string">`smart_factory.us_telemetry`</span>;
876</code></pre>
877<h4 id="cognitive-unification">COGNITIVE UNIFICATION</h4>
878<p><strong>The Multi-Region Model</strong></p>
879<ul>
880<li>The unified view now consumes from the abstraction layer, ensuring that changes to raw storage don&#39;t break the views down stream. </li>
881</ul>
882<pre><code class="lang-sql"><span class="hljs-comment">-- AI Instruction</span>
883"<span class="hljs-keyword">Create</span> a <span class="hljs-keyword">view</span> <span class="hljs-keyword">with</span> <span class="hljs-keyword">name</span>
884smart_factory.vw_telemetry that creates a <span class="hljs-keyword">union</span> <span class="hljs-keyword">of</span> all the <span class="hljs-keyword">fields</span> <span class="hljs-keyword">from</span> the views vw_[region]_telemetry. The regions <span class="hljs-keyword">include</span> us <span class="hljs-keyword">and</span> mx. <span class="hljs-keyword">List</span> <span class="hljs-keyword">out</span> all the <span class="hljs-keyword">field</span> names. <span class="hljs-keyword">Never</span> <span class="hljs-keyword">
884use</span> * <span class="hljs-keyword">for</span> <span class="hljs-keyword">field</span> <span class="hljs-keyword">names</span><span class="hljs-string">"</span>
885</code></pre>
886<p><strong>Unified Global View</strong></p>
887<pre><code class="lang-sql"><span class="hljs-comment">-- Semantic Abstraction Layer</span>
888<span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">OR</span> <span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">VIEW</span> <span class="hljs-string">`smart_factory.vw_telemetry`</span> <span class="hljs-keyword">AS</span>
889<span class="hljs-keyword">SELECT</span> 
890  device_number,
891  bay_id,
892  factory,
893  created
894<span class="hljs-keyword">FROM</span> <span class="hljs-string">`smart_factory.vw_us_telemetry`</span>
895<span class="hljs-keyword">UNION</span> ALL
896<span class="hljs-keyword">SELECT</span> 
897  device_number,
898  bay_id,
899  factory,
900  created
901<span class="hljs-keyword">FROM</span> <span class="hljs-string">`smart_factory.vw_mx_telemetry`</span>
902</code></pre>
903<h4 id="scaling-to-china-factory">SCALING TO CHINA FACTORY</h4>
904<p><strong>Evolving the Model</strong></p>
905<ul>
906<li>Adding the new China feed by generating the External Table definition via AI. </li>
907</ul>
908<pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">OR</span> <span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">EXTERNAL</span> <span class="hljs-keyword">TABLE</span> <span class="hljs-string">`smart_factory.cn_telemetry`</span> (
909  device_number <span class="hljs-keyword">STRING</span>,
910  bay_id INT64,
911  factory <span class="hljs-keyword">STRING</span>,
912  created <span class="hljs-keyword">STRING</span>
913)
914OPTIONS (
915  <span class="hljs-keyword">format</span> = <span class="hljs-string">'CSV'</span>,
916  uris = [<span class="hljs-string">'gs://factory-dl/cn/dev-900/telemetry*.csv.gz'</span>],
917  skip_leading_rows = <span class="hljs-number">1</span>,
918  compression = <span class="hljs-string">'GZIP'</span>
919</code></pre>
920<p><strong>Human-in-the-Loop DevOps</strong></p>
921<ul>
922<li>Use AI to update the unified view with the new data feed.  Review and apply the changes by the DevOps team, as changes to a production view require approval.</li>
923</ul>
924<h4 id="manufacturing-spc-root-cause-analysis">Manufacturing SPC &amp; Root Cause Analysis</h4>
925<ul>
926<li>This query calculates a rolling mean and standard deviation over the last 10 minutes of telemetry to detect anomalies, &#8220;Out of Control&#8221; conditions.</li>
927</ul>
928<pre><code class="lang-sql">WITH TelemetryStats AS (
929  <span class="hljs-keyword">SELECT</span>
930    machine_id,
931    <span class="hljs-keyword">timestamp</span>,
932    sensor_reading,
933    <span class="hljs-comment">-- Calculate rolling stats for the "Control Chart"</span>
934    <span class="hljs-keyword">AVG</span>(sensor_reading) <span class="hljs-keyword">OVER</span>(<span class="hljs-keyword">PARTITION</span> <span class="hljs-keyword">BY</span> machine_id <span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">timestamp</span> <span class="hljs-keyword">ROWS</span> <span class="hljs-keyword">BETWEEN</span> <span class="hljs-number">20</span> <span class="hljs-keyword">PRECEDING</span> <span class="hljs-keyword">AND</span> <span class="hljs-keyword">CURRENT</span> <span class="hljs-keyword">ROW</span>) <span class="hljs-keyword">as</span> rolling_avg,
935    <span class="hljs-keyword">STDDEV</span>(sensor_reading) <span class="hljs-keyword">OVER</span>(<span class="hljs-keyword">PARTITION</span> <span class="hljs-keyword">BY</span> machine_id <span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">timestamp</span> <span class="hljs-keyword">ROWS</span> <span class="hljs-keyword">BETWEEN</span> <span class="hljs-number">20</span> <span class="hljs-keyword">PRECEDING</span> <span class="hljs-keyword">AND</span> <span class="hljs-keyword">CURRENT</span> <span class="hljs-keyword">ROW</span>) <span class="hljs-keyword">as</span> rolling_stddev
936  <span class="hljs-keyword">FROM</span> <span class="hljs-string">`production_data.mx_telemetry_stream`</span>
937  <span class="hljs-keyword">WHERE</span> <span class="hljs-keyword">timestamp</span> &gt; TIMESTAMP_SUB(<span class="hljs-keyword">CURRENT_TIMESTAMP</span>(), <span class="hljs-built_in">INTERVAL</span> <span class="hljs-number">1</span> <span class="hljs-keyword">HOUR</span>)
938),
939Anomalies <span class="hljs-keyword">AS</span> (
940  <span class="hljs-keyword">SELECT</span> *,
941    <span class="hljs-comment">-- Define "Out of Control" (Reading &gt; 3 Sigma from mean)</span>
942    <span class="hljs-keyword">ABS</span>(sensor_reading - rolling_avg) &gt; (<span class="hljs-number">3</span> * rolling_stddev) <span class="hljs-keyword">AS</span> is_out_of_control
943  <span class="hljs-keyword">FROM</span> TelemetryStats
944)
945<span class="hljs-keyword">SELECT</span> * <span class="hljs-keyword">FROM</span> Anomalies <span class="hljs-keyword">WHERE</span> is_out_of_control = <span class="hljs-literal">TRUE</span>;
946</code></pre>
947<h4 id="control-chart-visualization">Control Chart Visualization</h4>
948<p><img alt="The Cognitive Data Lakehouse: AI-Driven Unification and Semantic Modeling in a Zero-ETL Environment - Control Charts" loading="lazy" src="https://www.ozkary.dev/assets/2026/ozkary-the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment-control-charts.png"></p>
949<h4 id="advantage-comparison-matrix">ADVANTAGE COMPARISON MATRIX</h4>
950<table>
951<thead>
952<tr>
953<th style="text-align:left">Metric</th>
954<th style="text-align:left">Manual Data Engineering</th>
955<th style="text-align:left">AI-Augmented Zero-ETL</th>
956</tr>
957</thead>
958<tbody>
959<tr>
960<td style="text-align:left">Unification Speed</td>
961<td style="text-align:left">Days/Weeks per Source</td>
962<td style="text-align:left">Minutes via Generative AI</td>
963</tr>
964<tr>
965<td style="text-align:left">Schema Drift</td>
966<td style="text-align:left">Manual Script Rewrites</td>
967<td style="text-align:left">Adaptive AI View Discovery</td>
968</tr>
969<tr>
970<td style="text-align:left">
970Infrastructure Cost</td>
971<td style="text-align:left">High (Data Redundancy)</td>
972<td style="text-align:left">Minimal (In-place on GCS)</td>
973</tr>
974</tbody>
975</table>
976<p><strong>Strategic Intelligence ROI:</strong></p>
977<blockquote>
978<p>ROI(ai) = Insights Velocity / (Movement Cost + Labor Hours)</p>
979</blockquote>
980<h4 id="final-thoughts-strategic-summary">FINAL THOUGHTS: STRATEGIC SUMMARY</h4>
981<p><strong>Legacy Challenges</strong></p>
982<ul>
983<li>Brittle ETL: Manual pipelines break with every schema change.</li>
984<li>Cost Inefficiency: Redundant storage for processed data.</li>
985<li>Semantic Silos: Hard-coded aliases for disparate naming conventions.</li>
986<li>Slow Time-to-Insight: Weeks spent on manual schema alignment.</li>
987</ul>
988<p><strong>AI-Assisted Solutions</strong></p>
989<ul>
990<li>Zero-ETL Arch: Cost-effective storage with Data Lake virtual access.</li>
991<li>Automated Inference: Vertex AI handles the &quot;heavy lifting&quot; of mapping.</li>
992<li>Adaptive DevOps: Scalable model evolution (USA &#8594; MEX &#8594; China).</li>
993<li>Unified Intelligence: One virtual source of truth for global analytics. </li>
994</ul>
995<blockquote>
996<p>Moving from Data Reporting to Active Semantic Intelligence.</p>
997</blockquote>
998<h3 id="we-ve-covered-a-lot-today-but-this-is-just-the-beginning-">We&#39;ve covered a lot today, but this is just the beginning!</h3>
999<p>If you&#39;re interested in learning more about building cloud data pipelines, I encourage you to check out my book, &#39;Data Engineering Process Fundamentals,&#39; part of the Data Engineering Process Fundamentals series. It provides in-depth explanations, code samples, and practical exercises to help in your learning.</p>
1000<p> <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-cover.jpg"></a>  <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-back-cover.jpg"></a></p>
1001<hr>
1002<h3 id="-upcoming-sessions">📅 Upcoming Sessions</h3>
1003<p>Our upcoming series expands beyond data engineering to bridge the gap between <strong>AI</strong>, <strong>Machine Learning</strong>, and <strong>modern cloud architecture</strong>. Using our <a href="https://github.com/ozkary/data-engineering-mta-tur
1003nstile">Data</a>, <a href="https://github.com/ozkary/ai-engineering">AI</a>, and <a href="https://github.com/ozkary/machine-learning-engineering">ML</a> GitHub blueprints, we provide the code-first patterns needed to build everything from Zero-ETL pipelines to scalable LLM-powered systems. Join us to explore how these integrated disciplines work together to turn raw data into production-ready intelligence.</p>
1004<hr>
1005<h3 id="-let-s-connect-build-together">🌟 Let&#39;s Connect &amp; Build Together</h3>
1006<p>If you enjoyed these resources, let&#39;s stay in touch! I share deep-dives into AI/ML patterns and host community events here:</p>
1007<ul>
1008<li><strong><a href="https://gdg.community.dev/gdg-broward-county-fl/">GDG Broward</a></strong>: Join our local dev community for meetups and workshops.</li>
1009<li><strong><a href="https://www.linkedin.com/in/ozkary">LinkedIn</a></strong>: Let&#39;s connect professionally! I share insights on engineering.</li>
1010<li><strong><a href="https://github.com/ozkary">GitHub</a></strong>: Follow my open-source journey and star the repos you find useful.</li>
1011<li><strong><a href="https://www.youtube.com/@ozkary">YouTube</a></strong>: Watch step-by-step tutorials on the projects listed above.</li>
1012<li><strong><a href="https://bsky.app/profile/ozkary.bsky.social">BlueSky</a></strong> / <strong><a href="https://x.com/ozkary">X / Twitter</a></strong>: Daily tech updates and quick engineering tips.</li>
1013</ul>
1014<p>👉 <em>Originally published at <a href="https://www.ozkary.com">ozkary.com</a></em></p>
1015</div>
1016<div class='post-teaser' style='display: flex; gap: 15px; flex-wrap: wrap; margin-bottom: 15px;'>
1017<div class='teaser-thumbnail' style='max-width: 200px; flex: 1 1 150px;'>
1018<a href='https://www.ozkary.com/2026/01/the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment.html'>
1019<img alt='Post Thumbnail' itemprop='image' loading='lazy' src='https://www.ozkary.dev/assets/2026/ozkary-the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment-sm.png' style='width: 100%; height: auto; border-radius: 6px; object-fit: cover;'/>
1020</a>
1021</div>
1022<div class='teaser-snippet' style='flex: 2 1 250px;'>
1023<p style='margin-bottom: 10px; color: #4a5568;'>
1024Overview  In the modern data landscape, the wall between &quot;where data lives&quot; and &quot;how we get insights&quot; is crumbling. This...
1025</p>
1026<a class='btn btn-sm btn-outline-primary' href='https://www.ozkary.com/2026/01/the-cognitive-data-lakehouse-ai-driven-unification-and-semantic-modeling-in-a-zero-etl-environment.html'>Read Article &rarr;</a>
1027</div>
1028</div>
1029<div class='clear'></div>
1030</div>
1031</article>
1032<article class='post hentry' itemprop='blogPost' itemscope='itemscope' itemtype='http://schema.org/BlogPosting'>
1033<header class='entry-header'>
1034<h1 class='post-title entry-title' itemprop='name'>
1035<a href='https://www.ozkary.com/2025/12/from-raw-data-to-governance-refining-data-medallion-architecture-dec-2025.html'>From Raw Data to Governance: Refining Data with the Medallion Architecture Dec 2025</a>
1036</h1>
1037<div class='entry-metapbt'>
1038<i class='fa fa-user'></i>&nbsp;<span itemprop='author' itemscope='itemscope' itemtype='https://schema.org/Person'><a href='https://www.ozkary.com/oscar-garcia-ozkary' itemprop='url'><span itemprop='name'>Oscar Garcia @ozkary</span></a></span>&nbsp;&nbsp;
1039                      <i class='fa fa-calendar'></i>
104012/10/2025
1041                      &nbsp;&nbsp;<i class='fa fa-clock-o'></i>
1042<span class='post-read-time' data-post-id='2199712317020061334'>Loading read time...</span>
1043</div>
1044</header>
1045<div class='post-body entry-content' id='post-body-2199712317020061334' itemprop='articleBody'>
1046<div id='post-body-full-2199712317020061334' style='display: none;'><h1 id="overview">Overview</h1>
1047<p>Build upon your existing data engineering expertise and discover how Medallion Architecture can transform your data strategy. This session provides a hands-on approach to implementing Medallion principles, empowering you to create a robust, scalable, and governed data platform.</p>
1048<p>We&#39;ll explore how to align data engineering processes with Medallion Architecture, identifying opportunities for optimization and improvement. By understanding the core principles and practical implementation steps, you&#39;ll learn how to optimize data pipelines, enhance data quality, and unlock valuable insights through a structured, layered approach to drive business success.</p>
1049<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture" loading="lazy" src="https://www.ozkary.dev/assets/2025/ozkary-from-raw-data-to-governance-medallion-architecture.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture"></p>
1050<ul>
1051<li>Follow this GitHub repo during the presentation: (Give it a star)</li>
1052</ul>
1053<blockquote>
1054<p>👉 <a href="https://github.com/ozkary/data-engineering-mta-tur
1054nstile">https://github.com/ozkary/data-engineering-mta-turnstile</a></p>
1055</blockquote>
1056<ul>
1057<li>Read more information on my blog at:  </li>
1058</ul>
1059<blockquote>
1060<p>👉 <a href="https://www.ozkary.com/2023/03/data-engineering-process-fundamentals.html">https://www.ozkary.com/2023/03/data-engineering-process-fundamentals.html</a></p>
1061</blockquote>
1062<h2 id="youtube-video">YouTube Video - Dec 2025</h2>
1063
1064<iframe width="560" height="315" src="https://www.youtube.com/embed/E87qPNObF7g?si=f6ii8FOVH8sPI0Dv" title="From Raw Data to Governance: Refining Data with the Medallion Architecture" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
1065
1066<h3 id="video-agenda">Video Agenda</h3>
1067<ul>
1068<li><p>Introduction to Medallion Architecture</p>
1069<ul>
1070<li>Defining Medallion Architecture</li>
1071<li>Core Principles</li>
1072<li>Benefits of Medallion Architecture</li>
1073</ul>
1074</li>
1075<li><p>The Raw Zone</p>
1076<ul>
1077<li>Understanding the purpose of the Raw Zone</li>
1078<li>Best practices for data ingestion and storage</li>
1079</ul>
1080</li>
1081<li><p>The Bronze Zone</p>
1082<ul>
1083<li>Data transformation and cleansing</li>
1084<li>Creating a foundation for analysis</li>
1085</ul>
1086</li>
1087<li><p>The Silver Zone</p>
1088<ul>
1089<li>Data optimization and summarization</li>
1090<li>Preparing data for consumption</li>
1091</ul>
1092</li>
1093<li><p>The Gold Zone</p>
1094<ul>
1095<li>Curated data for insights and action</li>
1096<li>Enabling self-service analytics</li>
1097</ul>
1098</li>
1099<li><p>Empowering Insights</p>
1100<ul>
1101<li>Data-driven decision-making</li>
1102<li>Accelerated Insights</li>
1103</ul>
1104</li>
1105<li><p>Data Governance</p>
1106<ul>
1107<li>Importance of data governance in Medallion Architecture</li>
1108<li>Implementing data ownership and stewardship</li>
1109<li>Ensuring data quality and security</li>
1110</ul>
1111</li>
1112</ul>
1113<p><strong>Why Attend:</strong></p>
1114<p>Gain a deep understanding of Medallion Architecture and its application in modern data engineering. Learn how to optimize data pipelines, improve data quality, and unlock valuable insights. Discover practical steps to implement Medallion principles in your organization and drive data-driven decision-making.</p>
1115<h2 id="presentation">Presentation</h2>
1116<h3 id="introducing-medallion-architecture">Introducing Medallion Architecture</h3>
1117<p>Medallion architecture is a data management approach that organizes data into distinct layers based on its quality and processing level.</p>
1118<ul>
1119<li><strong>Improved Data Quality:</strong> By separating data into different zones, you can focus on data quality at each stage.</li>
1120<li><strong>Enhanced Data Governance:</strong> Clear data ownership and lineage improve data trustworthiness.</li>
1121<li><strong>Accelerated Insights:</strong> Optimized data in the Silver and Gold zones enables faster query performance.</li>
1122<li><strong>Scalability:</strong> The layered approach can accommodate growing data volumes and complexity.</li>
1123<li><strong>Cost Efficiency:</strong> Optimized data storage and processing can reduce costs.</li>
1124</ul>
1125<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Design Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-high-level-design.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Design Diagram"></p>
1126<h3 id="the-raw-zone-foundation-of-your-data-lake">The Raw Zone: Foundation of Your Data Lake</h3>
1127<p>The Raw Zone is the initial landing place for raw, unprocessed data. It serves as a historical archive of your data sources.</p>
1128<ul>
1129<li><strong>Key Characteristics:</strong><ul>
1130<li>Unstructured or semi-structured format (e.g., CSV, JSON, Parquet)</li>
1131<li>Data is ingested as-is, without any cleaning or transformation</li>
1132<li>High volume and velocity</li>
1133<li>Data retention policies are crucial</li>
1134</ul>
1135</li>
1136<li><strong>Benefits:</strong><ul>
1137<li>Preserves original data for potential future analysis</li>
1138<li>Enables data reprocessing</li>
1139<li>Supports data lineage and auditability</li>
1140</ul>
1141</li>
1142</ul>
1143<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Raw Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-raw-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Raw Zone Diagram"></p>
1144<h3 id="the-bronze-zone-transforming-raw-data">The Bronze Zone: Transforming Raw Data</h3>
1145<p>The Bronze Zone is where raw data undergoes initial cleaning, structuring, and transformation. It serves as a staging area for data before moving to the Silver Zone.</p>
1146<ul>
1147<li><strong>Key Characteristics:</strong><ul>
1148<li>Data is cleansed and standardized</li>
1149<li>Basic transformations are applied (e.g., data type conversions, null handling)</li>
1150<li>Data is structured into tables or views</li>
1151<li>Data quality checks are implemented</li>
1152<li>Data retention policies may be shorter than the Raw Zone</li>
1153</ul>
1154</li>
1155<li><strong>Benefits:</strong><ul>
1156<li>Improves data quality and consistency</li>
1157<li>
1157Provides a foundation for further analysis</li>
1158<li>Enables data exploration and discovery</li>
1159</ul>
1160</li>
1161</ul>
1162<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Bronze Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-bronze-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Bronze Zone Diagram"></p>
1163<h3 id="the-silver-zone-a-foundation-for-insights">The Silver Zone: A Foundation for Insights</h3>
1164<p>The Silver Zone houses data that has been further refined, aggregated, and optimized for specific use cases. It serves as a bridge between the raw data and the final curated datasets.</p>
1165<ul>
1166<li><strong>Key Characteristics:</strong><ul>
1167<li>Data is cleansed, standardized, and enriched</li>
1168<li>Data is structured for analytical purposes (e.g., normalized, de-normalized)</li>
1169<li>Data is optimized for query performance (e.g., partitioning, indexing)</li>
1170<li>Data is aggregated and summarized for specific use cases</li>
1171</ul>
1172</li>
1173<li><strong>Benefits:</strong><ul>
1174<li>Improved query performance</li>
1175<li>Supports self-service analytics</li>
1176<li>Enables advanced analytics and machine learning</li>
1177<li>Reduces query costs</li>
1178</ul>
1179</li>
1180</ul>
1181<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Silver Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-silver-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Silver Zone Diagram"></p>
1182<h3 id="the-gold-zone-your-data-s-final-destination">The Gold Zone: Your Data&#39;s Final Destination</h3>
1183<ul>
1184<li><strong>Definition:</strong> The Gold Zone contains the final, curated datasets ready for consumption by business users and applications. It is the pinnacle of data transformation and optimization.</li>
1185<li><strong>Key Characteristics:</strong><ul>
1186<li>Data is highly refined, aggregated, and optimized for specific use cases</li>
1187<li>Data is often materialized for performance</li>
1188<li>Data is subject to rigorous quality checks and validation</li>
1189<li>Data is secured and governed</li>
1190</ul>
1191</li>
1192<li><strong>Benefits:</strong><ul>
1193<li>Enables rapid insights and decision-making</li>
1194<li>Supports self-service analytics and reporting</li>
1195<li>Provides a foundation for advanced analytics and machine learning</li>
1196<li>Reduces query latency</li>
1197</ul>
1198</li>
1199</ul>
1200<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Gold Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-gold-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Gold Zone Diagram"></p>
1201<h3 id="the-gold-zone-empowering-insights-and-actions">The Gold Zone: Empowering Insights and Actions</h3>
1202<p>The Gold Zone is the final destination for data, providing a foundation for insights, analysis, and action. It houses curated, optimized datasets ready for consumption.</p>
1203<ul>
1204<li><strong>Key Characteristics:</strong><ul>
1205<li>Data is accessible and easily consumable</li>
1206<li>Supports various analytical tools and platforms (BI, ML, data science)</li>
1207<li>Enables self-service analytics</li>
1208<li>Drives business decisions and actions</li>
1209</ul>
1210</li>
1211<li><strong>Examples of Consumption Tools:</strong><ul>
1212<li>Business Intelligence (BI) tools (Looker, Tableau, Power BI)</li>
1213<li>Data science platforms (Python, R, SQL)</li>
1214<li>Machine learning platforms (TensorFlow, PyTorch)</li>
1215<li>Advanced analytics tools</li>
1216</ul>
1217</li>
1218</ul>
1219<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Analysis Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-analysis.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Analysis Diagram"></p>
1220<h3 id="data-governance-the-cornerstone-of-data-management">Data Governance: The Cornerstone of Data Management</h3>
1221<p><strong>Data governance</strong> is the framework that defines how data is managed within an organization, while <strong>data management</strong> is the operational execution of those policies. Data Governance is essential for ensuring data quality, consistency, and security. </p>
1222<p><strong>Key components of data governance include:</strong></p>
1223<ul>
1224<li><strong>Data Lineage:</strong> Tracking data&#39;s journey from source to consumption. </li>
1225<li><strong>Data Ownership:</strong> Defining who is responsible for data accuracy and usage.</li>
1226<li><strong>Data Stewardship:</strong> Managing data on a day-to-day basis, ensuring quality and compliance.</li>
1227<li><strong>Data Security:</strong> Protecting data from unauthorized access, use, disclosure, disruption, modification, or destruction.</li>
1228<li><strong>Compliance:</strong> Adhering to industry regulations (e.g., GDPR, CCPA, HIPAA) and internal policies. </li>
1229</ul>
1230<p>By establishing clear roles, responsibilities, and data lineage, organizations can build trust in their data, improve decision-making, and mitigate risks. </p>
1231<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Data Governance " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-governance.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Data Governance"></p>
1232<h3 id="data-transformation-and-incremental-strategy">Data Transformation and Incremental Strategy</h3>
1233<p>The data transformation phase is a critical stage in a data warehouse project. This phase involves several key steps, including data extraction, cleaning, loading, data type casting, use of naming conventions, and implementing incremental loads to continuously insert the new information since the last update via batch processes.</p>
1234<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Data transformation lineage" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-lineage.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -   Data transformation lineage"></p>
1235<p>Data Lineage: Tracks the flow of data from its origin to its destination, including all the intermediate processes and transformations that it undergoes. </p>
1236<h3 id="data-governance-metadata">Data Governance : Metadata</h3>
1237<p>Assigns the owner, steward and responsibilities of the data.</p>
1238<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Governance Metadata " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-metadata.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Governance metadata"></p>
1239<h3 id="summary-leverage-medallion-architecture-for-success">Summary: Leverage Medallion Architecture for Success</h3>
1240<ul>
1241<li><strong>Key Benefits:</strong> <ul>
1242<li>Improved data quality</li>
1243<li>Enhanced governance</li>
1244<li>Accelerated insights</li>
1245<li>Scalability</li>
1246<li>
1246Cost Efficiency.</li>
1247</ul>
1248</li>
1249</ul>
1250<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-diagram.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Diagram"></p>
1251<h3 id="we-ve-covered-a-lot-today-but-this-is-just-the-beginning-">We&#39;ve covered a lot today, but this is just the beginning!</h3>
1252<p>If you&#39;re interested in learning more about building cloud data pipelines, I encourage you to check out my book, &#39;Data Engineering Process Fundamentals,&#39; part of the Data Engineering Process Fundamentals series. It provides in-depth explanations, code samples, and practical exercises to help in your learning.</p>
1253<p> <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-cover.jpg"></a>  <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-back-cover.jpg"></a></p>
1254<p><strong>Upcoming Talks:</strong></p>
1255<p>Join us for subsequent sessions in our Data Engineering Process Fundamentals series, where we will delve deeper into specific facets of data engineering, exploring topics such as data modeling, pipelines, and best practices in data governance.</p>
1256<p>This presentation is based on the book, <a href="https://www.amazon.com/Data-Engineering-Process-Fundamentals-Hands/dp/B0CV7TPSNB">Data Engineering Process Fundamentals</a>, which provides a more comprehensive guide to the topics we&#39;ll cover. You can find all the sample code and datasets used in this presentation on our popular GitHub repository <a href="https://github.com/ozkary/data-engineering-mta-tur
1256nstile">Introduction to Data Engineering Process Fundamentals</a>.</p>
1257<p>Thanks for reading! 😊 If you enjoyed this post and would like to stay updated with our latest content, don&#8217;t forget to follow us. Join our community and be the first to know about new articles, exclusive insights, and more!</p>
1258<ul>
1259<li><a href="https://gdg.community.dev/gdg-broward-county-fl/">Google Developer Group</a></li>
1260<li><a href="https://github.com/ozkary">GitHub</a></li>
1261<li><a href="https://x.com/ozkary">Twitter</a></li>
1262<li><a href="https://www.youtube.com/@ozkary">YouTube</a></li>
1263<li><a href="https://bsky.app/profile/ozkary.bsky.social">BlueSky</a></li>
1264</ul>
1265<p>👍 Originally published by <a href="https://www.ozkary.com">ozkary.com</a></p>
1266</div>
1267<div class='post-teaser' style='display: flex; gap: 15px; flex-wrap: wrap; margin-bottom: 15px;'>
1268<div class='teaser-thumbnail' style='max-width: 200px; flex: 1 1 150px;'>
1269<a href='https://www.ozkary.com/2025/12/from-raw-data-to-governance-refining-data-medallion-architecture-dec-2025.html'>
1270<img alt='Post Thumbnail' itemprop='image' loading='lazy' src='https://www.ozkary.dev/assets/2025/ozkary-from-raw-data-to-governance-medallion-architecture.png' style='width: 100%; height: auto; border-radius: 6px; object-fit: cover;'/>
1271</a>
1272</div>
1273<div class='teaser-snippet' style='flex: 2 1 250px;'>
1274<p style='margin-bottom: 10px; color: #4a5568;'>
1275Overview  Build upon your existing data engineering expertise and discover how Medallion Architecture can transform your data strategy. This...
1276</p>
1277<a class='btn btn-sm btn-outline-primary' href='https://www.ozkary.com/2025/12/from-raw-data-to-governance-refining-data-medallion-architecture-dec-2025.html'>Read Article &rarr;</a>
1278</div>
1279</div>
1280<div class='clear'></div>
1281</div>
1282</article>
1283<article class='post hentry' itemprop='blogPost' itemscope='itemscope' itemtype='http://schema.org/BlogPosting'>
1284<header class='entry-header'>
1285<h1 class='post-title entry-title' itemprop='name'>
1286<a href='https://www.ozkary.com/2025/11/from-raw-data-to-governance-refining-data-medallion-architecture.html'>From Raw Data to Governance: Refining Data with the Medallion Architecture Nov 2025</a>
1287</h1>
1288<div class='entry-metapbt'>
1289<i class='fa fa-user'></i>&nbsp;<span itemprop='author' itemscope='itemscope' itemtype='https://schema.org/Person'><a href='https://www.ozkary.com/oscar-garcia-ozkary' itemprop='url'><span itemprop='name'>Oscar Garcia @ozkary</span></a></span>&nbsp;&nbsp;
1290                      <i class='fa fa-calendar'></i>
129111/19/2025
1292                      &nbsp;&nbsp;<i class='fa fa-clock-o'></i>
1293<span class='post-read-time' data-post-id='3674577752927290015'>Loading read time...</span>
1294</div>
1295</header>
1296<div class='post-body entry-content' id='post-body-3674577752927290015' itemprop='articleBody'>
1297<div id='post-body-full-3674577752927290015' style='display: none;'><h1 id="overview">Overview</h1>
1298<p>Build upon your existing data engineering expertise and discover how Medallion Architecture can transform your data strategy. This session provides a hands-on approach to implementing Medallion principles, empowering you to create a robust, scalable, and governed data platform.</p>
1299<p>We&#39;ll explore how to align data engineering processes with Medallion Architecture, identifying opportunities for optimization and improvement. By understanding the core principles and practical implementation steps, you&#39;ll learn how to optimize data pipelines, enhance data quality, and unlock valuable insights through a structured, layered approach to drive business success.</p>
1300<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture" loading="lazy" src="https://www.ozkary.dev/assets/2025/ozkary-from-raw-data-to-governance-medallion-architecture.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture"></p>
1301<ul>
1302<li>Follow this GitHub repo during the presentation: (Give it a star)</li>
1303</ul>
1304<blockquote>
1305<p>👉 <a href="https://github.com/ozkary/data-engineering-mta-tur
1305nstile">https://github.com/ozkary/data-engineering-mta-turnstile</a></p>
1306</blockquote>
1307<ul>
1308<li>Read more information on my blog at:  </li>
1309</ul>
1310<blockquote>
1311<p>👉 <a href="https://www.ozkary.com/2023/03/data-engineering-process-fundamentals.html">https://www.ozkary.com/2023/03/data-engineering-process-fundamentals.html</a></p>
1312</blockquote>
1313<h2 id="youtube-video">YouTube Video</h2>
1314<iframe width="560" height="315" src="https://www.youtube.com/embed/EPYbPLKUxDE?si=sAe8k3beWEcxBEYT" title="From Raw Data to Governance: Refining Data with the Medallion Architecture" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
1315
1316<h3 id="video-agenda">Video Agenda</h3>
1317<ul>
1318<li><p>Introduction to Medallion Architecture</p>
1319<ul>
1320<li>Defining Medallion Architecture</li>
1321<li>Core Principles</li>
1322<li>Benefits of Medallion Architecture</li>
1323</ul>
1324</li>
1325<li><p>The Raw Zone</p>
1326<ul>
1327<li>Understanding the purpose of the Raw Zone</li>
1328<li>Best practices for data ingestion and storage</li>
1329</ul>
1330</li>
1331<li><p>The Bronze Zone</p>
1332<ul>
1333<li>Data transformation and cleansing</li>
1334<li>Creating a foundation for analysis</li>
1335</ul>
1336</li>
1337<li><p>The Silver Zone</p>
1338<ul>
1339<li>Data optimization and summarization</li>
1340<li>Preparing data for consumption</li>
1341</ul>
1342</li>
1343<li><p>The Gold Zone</p>
1344<ul>
1345<li>Curated data for insights and action</li>
1346<li>Enabling self-service analytics</li>
1347</ul>
1348</li>
1349<li><p>Empowering Insights</p>
1350<ul>
1351<li>Data-driven decision-making</li>
1352<li>Accelerated Insights</li>
1353</ul>
1354</li>
1355<li><p>Data Governance</p>
1356<ul>
1357<li>Importance of data governance in Medallion Architecture</li>
1358<li>Implementing data ownership and stewardship</li>
1359<li>Ensuring data quality and security</li>
1360</ul>
1361</li>
1362</ul>
1363<p><strong>Why Attend:</strong></p>
1364<p>Gain a deep understanding of Medallion Architecture and its application in modern data engineering. Learn how to optimize data pipelines, improve data quality, and unlock valuable insights. Discover practical steps to implement Medallion principles in your organization and drive data-driven decision-making.</p>
1365<h2 id="presentation">Presentation</h2>
1366<h3 id="introducing-medallion-architecture">Introducing Medallion Architecture</h3>
1367<p>Medallion architecture is a data management approach that organizes data into distinct layers based on its quality and processing level.</p>
1368<ul>
1369<li><strong>Improved Data Quality:</strong> By separating data into different zones, you can focus on data quality at each stage.</li>
1370<li><strong>Enhanced Data Governance:</strong> Clear data ownership and lineage improve data trustworthiness.</li>
1371<li><strong>Accelerated Insights:</strong> Optimized data in the Silver and Gold zones enables faster query performance.</li>
1372<li><strong>Scalability:</strong> The layered approach can accommodate growing data volumes and complexity.</li>
1373<li><strong>Cost Efficiency:</strong> Optimized data storage and processing can reduce costs.</li>
1374</ul>
1375<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Design Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-high-level-design.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Design Diagram"></p>
1376<h3 id="the-raw-zone-foundation-of-your-data-lake">The Raw Zone: Foundation of Your Data Lake</h3>
1377<p>The Raw Zone is the initial landing place for raw, unprocessed data. It serves as a historical archive of your data sources.</p>
1378<ul>
1379<li><strong>Key Characteristics:</strong><ul>
1380<li>Unstructured or semi-structured format (e.g., CSV, JSON, Parquet)</li>
1381<li>Data is ingested as-is, without any cleaning or transformation</li>
1382<li>High volume and velocity</li>
1383<li>Data retention policies are crucial</li>
1384</ul>
1385</li>
1386<li><strong>Benefits:</strong><ul>
1387<li>Preserves original data for potential future analysis</li>
1388<li>Enables data reprocessing</li>
1389<li>Supports data lineage and auditability</li>
1390</ul>
1391</li>
1392</ul>
1393<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Raw Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-raw-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Raw Zone Diagram"></p>
1394<h3 id="the-bronze-zone-transforming-raw-data">The Bronze Zone: Transforming Raw Data</h3>
1395<p>The Bronze Zone is where raw data undergoes initial cleaning, structuring, and transformation. It serves as a staging area for data before moving to the Silver Zone.</p>
1396<ul>
1397<li><strong>Key Characteristics:</strong><ul>
1398<li>Data is cleansed and standardized</li>
1399<li>Basic transformations are applied (e.g., data type conversions, null handling)</li>
1400<li>Data is structured into tables or views</li>
1401<li>Data quality checks are implemented</li>
1402<li>Data retention policies may be shorter than the Raw Zone</li>
1403</ul>
1404</li>
1405<li><strong>Benefits:</strong><ul>
1406<li>Improves data quality and consistency</li>
1407<li>
1407Provides a foundation for further analysis</li>
1408<li>Enables data exploration and discovery</li>
1409</ul>
1410</li>
1411</ul>
1412<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Bronze Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-bronze-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Bronze Zone Diagram"></p>
1413<h3 id="the-silver-zone-a-foundation-for-insights">The Silver Zone: A Foundation for Insights</h3>
1414<p>The Silver Zone houses data that has been further refined, aggregated, and optimized for specific use cases. It serves as a bridge between the raw data and the final curated datasets.</p>
1415<ul>
1416<li><strong>Key Characteristics:</strong><ul>
1417<li>Data is cleansed, standardized, and enriched</li>
1418<li>Data is structured for analytical purposes (e.g., normalized, de-normalized)</li>
1419<li>Data is optimized for query performance (e.g., partitioning, indexing)</li>
1420<li>Data is aggregated and summarized for specific use cases</li>
1421</ul>
1422</li>
1423<li><strong>Benefits:</strong><ul>
1424<li>Improved query performance</li>
1425<li>Supports self-service analytics</li>
1426<li>Enables advanced analytics and machine learning</li>
1427<li>Reduces query costs</li>
1428</ul>
1429</li>
1430</ul>
1431<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Silver Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-silver-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Silver Zone Diagram"></p>
1432<h3 id="the-gold-zone-your-data-s-final-destination">The Gold Zone: Your Data&#39;s Final Destination</h3>
1433<ul>
1434<li><strong>Definition:</strong> The Gold Zone contains the final, curated datasets ready for consumption by business users and applications. It is the pinnacle of data transformation and optimization.</li>
1435<li><strong>Key Characteristics:</strong><ul>
1436<li>Data is highly refined, aggregated, and optimized for specific use cases</li>
1437<li>Data is often materialized for performance</li>
1438<li>Data is subject to rigorous quality checks and validation</li>
1439<li>Data is secured and governed</li>
1440</ul>
1441</li>
1442<li><strong>Benefits:</strong><ul>
1443<li>Enables rapid insights and decision-making</li>
1444<li>Supports self-service analytics and reporting</li>
1445<li>Provides a foundation for advanced analytics and machine learning</li>
1446<li>Reduces query latency</li>
1447</ul>
1448</li>
1449</ul>
1450<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Gold Zone Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-gold-zone.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Gold Zone Diagram"></p>
1451<h3 id="the-gold-zone-empowering-insights-and-actions">The Gold Zone: Empowering Insights and Actions</h3>
1452<p>The Gold Zone is the final destination for data, providing a foundation for insights, analysis, and action. It houses curated, optimized datasets ready for consumption.</p>
1453<ul>
1454<li><strong>Key Characteristics:</strong><ul>
1455<li>Data is accessible and easily consumable</li>
1456<li>Supports various analytical tools and platforms (BI, ML, data science)</li>
1457<li>Enables self-service analytics</li>
1458<li>Drives business decisions and actions</li>
1459</ul>
1460</li>
1461<li><strong>Examples of Consumption Tools:</strong><ul>
1462<li>Business Intelligence (BI) tools (Looker, Tableau, Power BI)</li>
1463<li>Data science platforms (Python, R, SQL)</li>
1464<li>Machine learning platforms (TensorFlow, PyTorch)</li>
1465<li>Advanced analytics tools</li>
1466</ul>
1467</li>
1468</ul>
1469<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Analysis Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-analysis.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Analysis Diagram"></p>
1470<h3 id="data-governance-the-cornerstone-of-data-management">Data Governance: The Cornerstone of Data Management</h3>
1471<p><strong>Data governance</strong> is the framework that defines how data is managed within an organization, while <strong>data management</strong> is the operational execution of those policies. Data Governance is essential for ensuring data quality, consistency, and security. </p>
1472<p><strong>Key components of data governance include:</strong></p>
1473<ul>
1474<li><strong>Data Lineage:</strong> Tracking data&#39;s journey from source to consumption. </li>
1475<li><strong>Data Ownership:</strong> Defining who is responsible for data accuracy and usage.</li>
1476<li><strong>Data Stewardship:</strong> Managing data on a day-to-day basis, ensuring quality and compliance.</li>
1477<li><strong>Data Security:</strong> Protecting data from unauthorized access, use, disclosure, disruption, modification, or destruction.</li>
1478<li><strong>Compliance:</strong> Adhering to industry regulations (e.g., GDPR, CCPA, HIPAA) and internal policies. </li>
1479</ul>
1480<p>By establishing clear roles, responsibilities, and data lineage, organizations can build trust in their data, improve decision-making, and mitigate risks. </p>
1481<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Data Governance " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-governance.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Data Governance"></p>
1482<h3 id="data-transformation-and-incremental-strategy">Data Transformation and Incremental Strategy</h3>
1483<p>The data transformation phase is a critical stage in a data warehouse project. This phase involves several key steps, including data extraction, cleaning, loading, data type casting, use of naming conventions, and implementing incremental loads to continuously insert the new information since the last update via batch processes.</p>
1484<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Data transformation lineage" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-lineage.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -   Data transformation lineage"></p>
1485<p>Data Lineage: Tracks the flow of data from its origin to its destination, including all the intermediate processes and transformations that it undergoes. </p>
1486<h3 id="data-governance-metadata">Data Governance : Metadata</h3>
1487<p>Assigns the owner, steward and responsibilities of the data.</p>
1488<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Governance Metadata " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-metadata.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Governance metadata"></p>
1489<h3 id="summary-leverage-medallion-architecture-for-success">Summary: Leverage Medallion Architecture for Success</h3>
1490<ul>
1491<li><strong>Key Benefits:</strong> <ul>
1492<li>Improved data quality</li>
1493<li>Enhanced governance</li>
1494<li>Accelerated insights</li>
1495<li>Scalability</li>
1496<li>
1496Cost Efficiency.</li>
1497</ul>
1498</li>
1499</ul>
1500<p><img alt="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Diagram " loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-medallion-architecture-diagram.png" title="From Raw Data to Governance: Refining Data with the Medallion Architecture -  Medallion Architecture Diagram"></p>
1501<h3 id="we-ve-covered-a-lot-today-but-this-is-just-the-beginning-">We&#39;ve covered a lot today, but this is just the beginning!</h3>
1502<p>If you&#39;re interested in learning more about building cloud data pipelines, I encourage you to check out my book, &#39;Data Engineering Process Fundamentals,&#39; part of the Data Engineering Process Fundamentals series. It provides in-depth explanations, code samples, and practical exercises to help in your learning.</p>
1503<p> <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-cover.jpg"></a>  <a href="https://a.co/d/gyoRfbs"><img alt="Data Engineering Process Fundamentals - Book by Oscar Garcia" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-book-back-cover.jpg"></a></p>
1504<p><strong>Upcoming Talks:</strong></p>
1505<p>Join us for subsequent sessions in our Data Engineering Process Fundamentals series, where we will delve deeper into specific facets of data engineering, exploring topics such as data modeling, pipelines, and best practices in data governance.</p>
1506<p>This presentation is based on the book, <a href="https://www.amazon.com/Data-Engineering-Process-Fundamentals-Hands/dp/B0CV7TPSNB">Data Engineering Process Fundamentals</a>, which provides a more comprehensive guide to the topics we&#39;ll cover. You can find all the sample code and datasets used in this presentation on our popular GitHub repository <a href="https://github.com/ozkary/data-engineering-mta-tur
1506nstile">Introduction to Data Engineering Process Fundamentals</a>.</p>
1507<p>Thanks for reading! 😊 If you enjoyed this post and would like to stay updated with our latest content, don&#8217;t forget to follow us. Join our community and be the first to know about new articles, exclusive insights, and more!</p>
1508<ul>
1509<li><a href="https://gdg.community.dev/gdg-broward-county-fl/">Google Developer Group</a></li>
1510<li><a href="https://github.com/ozkary">GitHub</a></li>
1511<li><a href="https://x.com/ozkary">Twitter</a></li>
1512<li><a href="https://www.youtube.com/@ozkary">YouTube</a></li>
1513<li><a href="https://bsky.app/profile/ozkary.bsky.social">BlueSky</a></li>
1514</ul>
1515<p>👍 Originally published by <a href="https://www.ozkary.com">ozkary.com</a></p>
1516</div>
1517<div class='post-teaser' style='display: flex; gap: 15px; flex-wrap: wrap; margin-bottom: 15px;'>
1518<div class='teaser-thumbnail' style='max-width: 200px; flex: 1 1 150px;'>
1519<a href='https://www.ozkary.com/2025/11/from-raw-data-to-governance-refining-data-medallion-architecture.html'>
1520<img alt='Post Thumbnail' itemprop='image' loading='lazy' src='https://www.ozkary.dev/assets/2025/ozkary-from-raw-data-to-governance-medallion-architecture.png' style='width: 100%; height: auto; border-radius: 6px; object-fit: cover;'/>
1521</a>
1522</div>
1523<div class='teaser-snippet' style='flex: 2 1 250px;'>
1524<p style='margin-bottom: 10px; color: #4a5568;'>
1525Overview  Build upon your existing data engineering expertise and discover how Medallion Architecture can transform your data strategy. This...
1526</p>
1527<a class='btn btn-sm btn-outline-primary' href='https://www.ozkary.com/2025/11/from-raw-data-to-governance-refining-data-medallion-architecture.html'>Read Article &rarr;</a>
1528</div>
1529</div>
1530<div class='clear'></div>
1531</div>
1532</article>
1533<article class='post hentry' itemprop='blogPost' itemscope='itemscope' itemtype='http://schema.org/BlogPosting'>
1534<header class='entry-header'>
1535<h1 class='post-title entry-title' itemprop='name'>
1536<a href='https://www.ozkary.com/2025/10/from-raw-data-to-analytics-modern-data-layer-architecture.html'>From Raw Data to Analytics: The Modern Data Layer Architecture</a>
1537</h1>
1538<div class='entry-metapbt'>
1539<i class='fa fa-user'></i>&nbsp;<span itemprop='author' itemscope='itemscope' itemtype='https://schema.org/Person'><a href='https://www.ozkary.com/oscar-garcia-ozkary' itemprop='url'><span itemprop='name'>Oscar Garcia @ozkary</span></a></span>&nbsp;&nbsp;
1540                      <i class='fa fa-calendar'></i>
154110/29/2025
1542                      &nbsp;&nbsp;<i class='fa fa-clock-o'></i>
1543<span class='post-read-time' data-post-id='1354332841131979520'>Loading read time...</span>
1544</div>
1545</header>
1546<div class='post-body entry-content' id='post-body-1354332841131979520' itemprop='articleBody'>
1547<div id='post-body-full-1354332841131979520' style='display: none;'><h1 id="overview"><div class="separator" style="clear: both; text-align: center;"><br /></div><br />Overview</h1>
1548<p>This presentation is part of the Data Engineering Process Fundamentals series, focusing on the essential architectural components&#8212;the Data Lake and the Data Warehouse&#8212;and defining their respective roles in a modern analytics ecosystem.</p>
1549<p><img alt="From Raw Data to Analytics: The Modern Data Layer Architecture" loading="lazy" src="https://www.ozkary.dev/assets/2025/ozkary-data-engineering-data-lake-data-warehouse-raw-data-to-analytics.png" title="From Raw Data to Analytics: The Modern Data Layer Architecture" /></p>
1550<ul>
1551<li>Follow this GitHub repo during the presentation: (Star the project to follow and get updates)</li>
1552</ul>
1553<blockquote>
1554<p>👉 <a href="https://github.com/ozkary/data-engineering-mta-tur
1554nstile">GitHub Repo</a></p>
1555</blockquote>
1556<ul>
1557<li>Data engineering Series:  </li>
1558</ul>
1559<blockquote>
1560<p>👉 <a href="https://www.ozkary.com/2023/03/data-engineering-process-fundamentals.html">Blog Series</a></p>
1561</blockquote>
1562<h2 id="youtube-video">YouTube Video</h2>
1563<iframe allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen="" frameborder="0" height="315" referrerpolicy="strict-origin-when-cross-origin" src="https://www.youtube.com/embed/9DYtKz4qVbA?si=mk_s1myV6aB7bBgZ" title="From Raw Data to Analytics: The Modern Data Layer Architecture" width="560"></iframe>
1564
1565<h3 id="video-agenda">Video Agenda</h3>
1566<p>Agenda:</p>
1567<ol>
1568<li><p>Introduction to Data Engineering:</p>
1569</li>
1570<li><p>Brief overview of the data engineering landscape and its critical role in modern data-driven organizations.</p>
1571</li>
1572<li><p>Operational Data</p>
1573</li>
1574<li><p>Understanding Data Lakes:</p>
1575</li>
1576<li><p>Explanation of what a data lake is and its purpose in storing vast amounts of raw and unstructured data.</p>
1577</li>
1578<li><p>Exploring Data Warehouses:</p>
1579</li>
1580<li><p>Definition of data warehouses and their role in storing structured, processed, and business-ready data.</p>
1581</li>
1582<li><p>Comparing Data Lakes and Data Warehouses:</p>
1583</li>
1584<li><p>Comparative analysis of data lakes and data warehouses, highlighting their strengths and weaknesses.</p>
1585</li>
1586<li><p>Discussing when to use each based on specific use cases and business needs.</p>
1587</li>
1588<li><p>Integration and Data Pipelines:</p>
1589</li>
1590<li><p>Insight into the seamless integration of data lakes and data warehouses within a data engineering pipeline.</p>
1591</li>
1592<li><p>Code walkthrough showcasing data movement and transformation between these two crucial components.</p>
1593</li>
1594<li><p>Real-world Use Cases:</p>
1595</li>
1596<li><p>Presentation of real-world use cases where effective use of data lakes and data warehouses led to actionable insights and business success.</p>
1597</li>
1598<li><p>Hands-on demonstration using Python, Jupyter Notebook and SQL to solidify the concepts discussed, providing attendees with practical insights and skills.</p>
1599</li>
1600<li><p>Q&amp;A and Hands-on Session:</p>
1601</li>
1602<li><p>An interactive Q&amp;A session to address any queries.</p>
1603</li>
1604</ol>
1605<p>Conclusion:</p>
1606<p>This session aims to equip attendees with a strong foundation in data engineering, focusing on the pivotal role of data lakes and data warehouses. By the end of this presentation, participants will grasp how to effectively utilize these tools, enabling them to design efficient data solutions and drive informed business decisions.</p>
1607<p>This presentation will be accompanied by live code demonstrations and interactive discussions, ensuring attendees gain practical knowledge and valuable insights into the dynamic world of data engineering.</p>
1608<h3 id="supporting-materials-reminder">Supporting Materials Reminder</h3>
1609<p><strong>Subsequent Sessions:</strong> Join us for future sessions in our Data Engineering Process Fundamentals series, where we will build a data pipeline and delve deeper into topics like orchestration and governance.</p>
1610<p><strong>Resources:</strong> This presentation is based on the book, Data Engineering Process Fundamentals, and all supporting code and examples are available on our popular GitHub repository.</p>
1611<h2 id="presentation">Presentation</h2>
1612<h3 id="data-engineering-overview">Data Engineering Overview</h3>
1613<p>A Data Engineering Process involves executing steps to understand the problem, scope, design, and architecture for creating a solution. This enables ongoing big data analysis using analytical and visualization tools.</p>
1614<p><strong>Topics</strong></p>
1615<ul>
1616<li>Data Lake and Data Warehouse</li>
1617<li>Discovery and Data Analysis</li>
1618<li>Design and Infrastructure Planning</li>
1619<li>Data Lake - Pipeline and Orchestration</li>
1620<li>Data Warehouse - Design and Implementation</li>
1621<li>Analysis and Visualization</li>
1622</ul>
1623<p><strong>Follow this project: Give a star</strong></p>
1624<blockquote>
1625<p>
1625👉 <a href="//github.com/ozkary/data-engineering-mta-turnstile">Data Engineering Process Fundamentals</a></p>
1626</blockquote>
1627<h3 id="operational-data">Operational Data</h3>
1628<p>Operational data is often generated by applications, and it is stored in transactional relational databases like SQL Server, Oracle and NoSQL (JSON) databases like MongoDB, Firebase. This is the data that is created after an application saves a user transaction like contact information, a purchase or other activities that are available from the application. </p>
1629<p><strong>Features:</strong></p>
1630<ul>
1631<li>Application support and transactions</li>
1632<li>Relational data structure and SQL or document structure NoSQL</li>
1633<li>Small queries for case analysis</li>
1634</ul>
1635<p><strong>Not Best For:</strong></p>
1636<ul>
1637<li>Reporting system</li>
1638<li>Large queries</li>
1639<li>Centralized Big Data system</li>
1640</ul>
1641<p><img alt="Data Engineering Process Fundamentals - Operational Data" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-operational-data.png" title="Data Engineering Process Fundamentals - Operational Data" /></p>
1642<h3 id="data-lake-analytical-data-staging">Data Lake - Analytical Data Staging</h3>
1643<p>A Data Lake is an optimized storage system for Big Data scenarios. The primary function is to store the data in its raw format without any transformation. Analytical data is the transaction data that has been extracted from a source system via a data pipeline as part of the staging data process.</p>
1644<p><strong>Features:</strong></p>
1645<ul>
1646<li>Store the data in its raw format without any transformation </li>
1647<li>This can include structure data like CSV files, unstructured data like JSON and XML documents, or column-base data like parquet files</li>
1648<li>Low Cost for massive storage power</li>
1649<li>Not Designed for querying or data analysis</li>
1650<li>It is used as external tables by most systems</li>
1651</ul>
1652<p><img alt="Data Engineering Process Fundamentals - Analytical Data staging" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-staging-data.png" /></p>
1653<h3 id="data-warehouse-analytical-data">Data Warehouse - Analytical Data</h3>
1654<p>A Data Warehouse is a centralized storage system that stores integrated data from multiple sources. The system is designed to host and serve Big Data scenarios with lower operational cost than transaction databases, but higher costs than a Data Lake. This system host the Analytical Data that has been processed and is ready for analytical purposes.</p>
1655<p><strong>Data Warehouse Features:</strong></p>
1656<ul>
1657<li>Stores historical data in relational tables with an optimized schema, which enables the data analysis process</li>
1658<li>Provides SQL support to query the data</li>
1659<li>It can integrate external resources like CSV and parquet files that are stored on Data Lakes as external tables</li>
1660<li>The system is designed to host and serve Big Data scenarios. It is not meant to be used as a transactional system</li>
1661<li>Storage is more expensive</li>
1662<li>Offloads archived data to Data Lakes</li>
1663</ul>
1664<p><img alt="Data Engineering Process Fundamentals - Analytical Data Store" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-analytical-data.png" /></p>
1665<h3 id="discovery-data-analysis">Discovery - Data Analysis</h3>
1666<p>During the discovery phase of a Data Engineering Process, we look to identify and clearly document a problem statement, which helps us have an understanding of what we are trying to solve. We also look at our analytical approach to make observations about at the data, its structure and source. This leads us into defining the requirements for the project, so we can define the scope, design and architecture of the solution.</p>
1667<ul>
1668<li>Download sample data files</li>
1669<li>Run experiments to make observations</li>
1670<li>Write Python scripts using VS Code or Jupyter Notebooks</li>
1671<li>Transform the data with Pandas</li>
1672<li>Make charts with Plotly</li>
1673<li>Document the requirements</li>
1674</ul>
1675<p><img alt="Data Engineering Process Fundamentals - Data Analysis and discovery" loading="lazy" src="https://www.ozkary.dev/assets/2023/ozkary-data-engineering-process-jupyter-read-file.png" /></p>
1676<h3 id="design-and-planning">Design and Planning</h3>
1677<p>The design and planning phase of a data engineering project is crucial for laying out the foundation of a successful system. It involves defining the system architecture, designing data pipelines, implementing source control practices, ensuring continuous integration and deployment (CI/CD), and leveraging tools like Docker and Terraform for infrastru
1677cture automation.</p>
1678<ul>
1679<li>Use GitHub for code repo and for CI/CD actions</li>
1680<li>Use Terraform is an Infrastructure as Code (IaC) tool that enables us to manage cloud resources across multiple cloud providers</li>
1681<li>Use Docker containers to run the code and manage its dependencies</li>
1682</ul>
1683<p><img alt="Data Engineering Process Fundamentals - Design and Planning" loading="lazy" src="https://www.ozkary.dev/assets/2023/ozkary-data-engineering-design-terraform-docker.png" /></p>
1684<h3 id="data-lake-pipeline-and-orchestration">Data Lake - Pipeline and Orchestration</h3>
1685<p>A data pipeline is basically a workflow of tasks that can be executed in Docker containers. The execution, scheduling, managing and monitoring of the pipeline is referred to as orchestration. In order to support the operations of the pipeline and its orchestration, we need to provision a VM and data lake, and monitor cloud resources. </p>
1686<ul>
1687<li>This can be code-centric, leveraging languages like Python</li>
1688<li>Or a low-code approach, utilizing tools such as Azure Data Factory, which provides a turn-key solution</li>
1689<li>Monitor services enable us to track telemetry data</li>
1690<li>Docker Hub, GitHub can be used for the CI/CD process</li>
1691</ul>
1692<p><img alt="Data Engineering Process Fundamentals - Data Lake - Data Pipeline and Orchestration" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-data-pipeline.png" /></p>
1693<h3 id="data-warehouse-design-and-implementation">Data Warehouse - Design and Implementation</h3>
1694<p>In the design phase, we lay the groundwork by defining the database system, schema model, and technology stack required to support the data warehouse&#8217;s implementation and operations. In the implementation phase, we focus on converting conceptual data models into a functional system. By creating concrete structures like dimension and fact tables and performing data transformation tasks, including data cleansing, integration, and scheduled batch loading, we ensure that raw data is processed and unified for analysis. Create a repeatable and extendable process.</p>
1695<p><img alt="Data Engineering Process Fundamentals - Data Warehouse Design and Implementation" loading="lazy" src="https://www.ozkary.dev/assets/2023/ozkary-data-engineering-process-data-warehouse-design.png" /></p>
1696<h3 id="data-warehouse-data-analysis">Data Warehouse - Data Analysis</h3>
1697<p>Data analysis is the practice of exploring data and understanding its meaning. It involves activities that can help us achieve a specific goal, such as identifying data dimensions and measures, as well as data analysis to identify outliers, trends, and distributions. </p>
1698<ul>
1699<li>We can accomplish these activities by writing code using Python and Pandas, SQL, Visual Studio Code or Jupyter Notebooks. </li>
1700<li>What's more, we can use libraries, such as Plotly, to generate some visuals to further analyze data and create prototypes.</li>
1701</ul>
1702<p><img alt="Data Engineering Process Fundamentals - Data Analysis" loading="lazy" src="https://www.ozkary.dev/assets/2024/ozkary-data-engineering-process-fundamentals-data-analysis-code.png" /></p>
1703<h3 id="data-analysis-and-visualization">Data Analysis and Visualization</h3>
1704<p>Data visualization is a powerful tool that takes the insights derived from data analysis and presents them in a visual format. While tables with numbers on a report provide raw information, visualizations allow us to grasp complex relationships and trends at a glance. </p>
1705<ul>
1706<li>Dashboards, in particular, bring together various visual components like charts, graphs, and scorecards into a unified interface that can help us tell a story</li>
1707<li>Use tools like PowerBI, Looker, Tableau to model the data and create enterprise level visualizations</li>
1708</ul>
1709<p><img alt="Data Engineering Process Fundamentals - Data Visualization" loading="lazy" src="https://www.ozkary.dev/assets/2023/ozkary-data-engineering-process-data-analysis-visualization-dashboard.png" /></p>
1710<h3 id="conclusion">Conclusion</h3>
1711<p>Both data lakes and data warehouses are essential components of a data engineering project. The primary function of a data lake is to store large amounts of operational data in its raw format, serving as a staging area for analytical processes. In contrast, a data warehouse acts as a centralized repository for information, enabling engineers to transform, process, and store extensive data. This allows the analytical team to utilize coding languages like Python and tools such as Jupyter Notebooks, as well as low-code platforms like Looker Studio and Power BI, to create enterprise-quality dashboards for the organization.</p>
1712<p><strong>Upcoming Talks:</strong></p>
1713<p>Join us for subsequent sessions in our Data Engineering Process Fundamentals series, where we will delve deeper into specific facets of data engineering, exploring topics such as data modeling, pipelines, and best practices in data governance.</p>
1714<p>This presentation is based on the book, <a href="https://www.amazon.com/Data-Engineering-Process-Fundamentals-Hands/dp/B0CV7TPSNB">Data Engineering Process Fundamentals</a>, which provides a more comprehensive guide to the topics we'll cover. You can find all the sample code and datasets used in this presentation on our popular GitHub repository <a href="https://github.com/ozkary/data-engineering-mta-tur
1714nstile">Introduction to Data Engineering Process Fundamentals</a>.</p>
1715<p>Thanks for reading! 😊 If you enjoyed this post and would like to stay updated with our latest content, don&#8217;t forget to follow us. Join our community and be the first to know about new articles, exclusive insights, and more!</p>
1716<ul>
1717<li><a href="https://gdg.community.dev/gdg-broward-county-fl/">Google Developer Group</a></li>
1718<li><a href="https://github.com/ozkary">GitHub</a></li>
1719<li><a href="https://x.com/ozkary">Twitter</a></li>
1720<li><a href="https://www.youtube.com/@ozkary">YouTube</a></li>
1721<li><a href="https://bsky.app/profile/ozkary.bsky.social">BlueSky</a></li>
1722</ul>
1723<p>👍 Originally published by <a href="https://www.ozkary.com">ozkary.com</a></p>
1724</div>
1725<div class='post-teaser' style='display: flex; gap: 15px; flex-wrap: wrap; margin-bottom: 15px;'>
1726<div class='teaser-thumbnail' style='max-width: 200px; flex: 1 1 150px;'>
1727<a href='https://www.ozkary.com/2025/10/from-raw-data-to-analytics-modern-data-layer-architecture.html'>
1728<img alt='Post Thumbnail' itemprop='image' loading='lazy' src='https://www.ozkary.dev/assets/2025/ozkary-data-engineering-data-lake-data-warehouse-raw-data-to-analytics.png' style='width: 100%; height: auto; border-radius: 6px; object-fit: cover;'/>
1729</a>
1730</div>
1731<div class='teaser-snippet' style='flex: 2 1 250px;'>
1732<p style='margin-bottom: 10px; color: #4a5568;'>
1733Overview  This presentation is part of the Data Engineering Process Fundamentals series, focusing on the essential architectural components&#8212;...
1734</p>
1735<a class='btn btn-sm btn-outline-primary' href='https://www.ozkary.com/2025/10/from-raw-data-to-analytics-modern-data-layer-architecture.html'>Read Article &rarr;</a>
1736</div>
1737</div>
1738<div class='clear'></div>
1739</div>
1740</article>
1741</div>
1742<div class='blog-pager' id='blog-pager' style='display: flex; justify-content: space-between; margin-bottom: 25px;'>
1743<a class='btn btn-outline-primary' href='https://www.ozkary.com/'>&larr; Newer Posts</a>
1744<a class='btn btn-outline-primary' href='https://www.ozkary.com/search?updated-max=2025-10-29T16:11:00-04:00&max-results=7&start=5&by-date=false'>Older Posts &rarr;</a>
1745</div>
1746</div></div>
1747</main>
1748</div>
1749<aside class='widget-areapbt' id='secondarypbt' role='complementary'>
1750<div class='sidebarrightsec section' id='sidebarrightsec'><div class='widget HTML' data-version='1' id='HTML1'>
1751<h2 class='title'>Upcoming Free Events</h2>
1752<div class='widget-content'>
1753<style>
1754#toast-event-content a {color:#e2e8f1 !important; margin-bottom:10px !important; font-weight:400 !important}
1755</style>
1756<div class="events">
1757    <section class="events-section">
1758        <div id="next-events-links">
1759             <p style="color: #64748b; font-style: italic;">Loading new events...</p>
1760        </div>
1761    </section>
1762</div>
1763<script>
1764class RSSEventLoaderPanel {
1765    constructor(options = {}) {
1766        this.feedUrl = options.feedUrl || 'https://www.ozkary.dev/feed.xml';
1767        this.containerId = options.containerId || 'past-events-container';
1768        this.maxItems = options.maxItems || 5;
1769        // Uses CORS proxy fallback if fetching cross-origin directly fails
1770        this.corsProxy = 'https://api.allorigins.win/raw?url=';
1771      	this.showFutureOnly = options.showFutureOnly ?? false;
1772	this.widgetContainer = document.getElementById('HTML1');
1773        this.showToast = false;
1774    }
1775
1776    async init() {
1777        const container = document.getElementById(this.containerId);
1778        if (!container) {
1779          console.log(`container is not found ${this.containerId}`);
1780          return;
1781        } 
1782
1783        const isEventPage = window.location.pathname.includes('/p/event-');    
1784  	if (isEventPage && this.widgetContainer) {
1785      	  this.widgetContainer.style.display = 'none';
1786      	  return;
1787	}
1788
1789        try {
1790            const xmlDoc = await this.fetchFeed();
1791            const items = xmlDoc.querySelectorAll('item, entry');
1792            
1793            if (items.length === 0) {
1794                container.innerHTML = '<p>No events or articles found.</p>';
1795		this.widgetContainer.style.display = 'none';
1796                return;
1797            }
1798            this.showToast = true; 
1799
1800            let htmlOutput = '';
1801            const count = Math.min(items.length, this.maxItems);
1802
1803            for (let i = 0; i < count; i++) {
1804                const item = items[i];
1805                const cardData = this.parseItem(item);
1806              
1807              	if (this.showFutureOnly && !cardData.isFuture){
1808                  continue;
1809                }
1810              
1811                if (this.showCards){
1812                	htmlOutput += this.renderCard(cardData);
1813                } else {
1814                	htmlOutput += this.renderLink(cardData);
1815                }
1816            }
1817
1818            container.innerHTML = htmlOutput;
1819
1820	    const timer = setTimeout(() => {
1821      		clearTimeout(timer);
1822	        this.initEventToast();
1823	    }, 3500);
1824
1825
1826        } catch (error) {
1827            console.error('Error loading RSS feed:', error);
1828            container.innerHTML = '<p>Unable to load recent posts at this time.</p>';
1829        }
1830    }
1831
1832    async fetchFeed() {
1833        let response;
1834        try {
1835            // Attempt direct fetch first
1836            response = await fetch(this.feedUrl);
1837            if (!response.ok) throw new Error('Direct fetch failed');
1838        } catch (e) {
1839            // Fallback via CORS proxy if direct fetch is blocked
1840            response = await fetch(`${this.corsProxy}${encodeURIComponent(this.feedUrl)}`);
1841        }
1842
1843        const xmlText = await response.text();
1844        const parser = new DOMParser();
1845        return parser.parseFromString(xmlText, 'text/xml');
1846    }
1847
1848    parseItem(item) {
1849        // Extract Title
1850        const title = item.querySelector('title')?.textContent || 'Untitled Event';
1851
1852        // Extract Link
1853        let link = item.querySelector('link')?.textContent;
1854        if (!link) {
1855            link = item.querySelector('link')?.getAttribute('href') || '#';
1856        }
1857
1858        // Extract Date and Format
1859        const rawDate = item.querySelector('pubDate, updated, date')?.textContent;
1860        const formattedDate = rawDate ? this.formatDate(rawDate) : 'Recent';
1861
1862        // Extract Description and Clean HTML Tags
1863      	let summary = item.querySelector('summary')?.textContent || '';
1864      
1865        let description = item.querySelector('description, content')?.textContent || '';
1866        const tempDiv = document.createElement('div');
1867        tempDiv.innerHTML = description;
1868        
1869        // Try extracting an image from description if no media tag exists
1870        let imageUrl = item.querySelector('enclosure')?.getAttribute('url') || 
1871                         item.querySelector('media\\:content, content')?.getAttribute('url') || 
1872                         tempDiv.querySelector('img')?.getAttribute('src') || 
1873                         'https://www.ozkary.dev/assets/default-event.jpg';
1874
1875        // Clean description text for card summary
1876        let cleanText = summary || tempDiv.textContent || tempDiv.innerText || '';
1877        cleanText = cleanText.trim().substring(0, 275) + '...';
1878
1879        const isFuture = this.isFuture(formattedDate);
1880      	
1881        return { title, link, formattedDate, description: cleanText, imageUrl, isFuture };
1882    }
1883  
1884  	isFuture(dateString) {
1885    	const date = new Date(dateString);
1886        if (isNaN(date.getTime())) return false;      
1887      
1888      	const today = Date.now();      	
1889		return (date - today) > -1;      
1890    }
1891
1892    formatDate(dateString) {
1893        const date = new Date(dateString);
1894        if (isNaN(date.getTime())) return dateString;
1895        return date.toLocaleDateString('en-US', {
1896            month: 'short',
1897            day: 'numeric',
1898            year: 'numeric'
1899        });
1900    }
1901     
1902  // Example helper method inside your RSSEventLoader class
1903  renderLink({ title, link, formattedDate, description, imageUrl, isFuture }) {
1904    return `
1905      <li style="border-bottom: 1px solid #f1f5f9; padding-bottom: 10px; margin-bottom: 10px; list-style-type: none;">
1906        <a href="${link}" 
1907           style="display: block; font-size: 0.85rem; font-weight: 600; color: #0f172a; text-decoration: none; line-height: 1.3; margin-bottom: 4px;">
1908          ${title}
1909        </a>
1910        <div style="display: flex; align-items: center; justify-content: space-between; margin-top: 6px;">
1911          <span style="font-size: 0.75rem; color: #64748b; font-weight: 500;">
1912            🗓&#65039; ${formattedDate}
1913          </span>
1914          <a href="${link}" 
1915             style="display: inline-block; background-color: #3d4144; color: #fff !important; font-size: 0.7rem; font-weight: 700; text-decoration: none; padding: 3px 8px; border-radius: 4px; text-transform: uppercase;">
1916            Register
1917          </a>
1918        </div>
1919      </li>
1920    `;
1921  }
1922
1923 async initEventToast() { 
1924  
1925  if (!this.showToast) return;
1926  if (sessionStorage.getItem('eventToastDismissed') === 'true') return;
1927
1928  const sourceElement = document.getElementById('next-events-links');
1929  
1930  if (!sourceElement || !sourceElement.innerHTML.trim()) {
1931    return;
1932  }
1933
1934  const innerContentText = sourceElement.innerText || sourceElement.textContent;
1935  if (/no events/i.test(innerContentText)) {
1936    // Hide the master widget container from the DOM entirely
1937    if (this.widgetContainer) {
1938      this.widgetContainer.style.display = 'none';
1939    }
1940    return;
1941  }
1942  
1943  const toast = document.createElement('div');
1944  toast.id = 'mobile-event-toast';
1945  
1946  // Mobile-optimized floating container styles
1947  toast.style.cssText = `
1948    position: fixed;
1949    bottom: -150px;
1950    left: 50%;
1951    transform: translateX(-50%);
1952    width: 92%;
1953    max-width: 400px;
1954    background: #0f172a;
1955    border: 1px solid #334155;
1956    border-radius: 12px;
1957    padding: 12px 14px;
1958    box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
1959    z-index: 99999;
1960    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
1961    transition: bottom 0.4s cubic-bezier(0.175, 0.885, 0.32, 1.275);
1962    box-sizing: border-box;
1963  `;
1964
1965  toast.innerHTML = `
1966    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px; border-bottom: 1px solid #334155; padding-bottom: 6px;">
1967      <span style="font-size: 0.7rem; text-transform: uppercase; color: #e2e8f1; font-weight: 700; letter-spacing: 0.5px; display: flex; align-items: center; gap: 4px;">
1968        &#9889; Upcoming Free Event
1969      </span>
1970      <button id="close-event-toast" 
1971              style="background: transparent; border: none; color: #94a3b8; font-size: 1.2rem; cursor: pointer; padding: 0 4px; line-height: 1;">
1972        &times;
1973      </button>
1974    </div>
1975    <div id="toast-event-content"></div>
1976  `;
1977
1978  document.body.appendChild(toast);
1979
1980  const toastContent = document.getElementById('toast-event-content');
1981  toastContent.appendChild(sourceElement.cloneNode(true));
1982
1983  document.getElementById('close-event-toast').addEventListener('click', () => {
1984    toast.style.bottom = '-200px';
1985    sessionStorage.setItem('eventToastDismissed', 'true');
1986  });
1987
1988  toast.style.bottom = '16px';
1989
1990}
1991
1992  
1993}
1994
1995// Auto-initialize when the page DOM is ready
1996document.addEventListener('DOMContentLoaded', () => {  
1997    
1998    // Fetch & Render Upcoming Events
1999    const sideLoader = new RSSEventLoaderPanel({
2000        feedUrl: 'https://www.ozkary.dev/events-feed.xml',
2001        containerId: 'next-events-links',
2002        maxItems: 3,
2003        showFutureOnly: true        
2004    });
2005        
2006    sideLoader.init();  
2007 
2008});
2009
2010</script>
2010
2011</div>
2012<div class='clear'></div>
2013</div><div class='widget HTML' data-version='1' id='HTML5'>
2014<h2 class='title'>Books on Amazon</h2>
2015<div class='widget-content'><div style="margin: 0 auto !important;display:block; text-align:center; align-items:center"><h5 style="font-size: 0.95rem; font-weight: bold;">Data Engineering Process Fundamentals</h5>
2016                <a href="https://www.amazon.com/Data-Engineering-Process-Fundamentals-Hands/dp/B0CV7TPSNB" target="_blank" rel="noopener">
2017                <img src="https://m.media-amazon.com/images/I/41+MsbOIV1L.jpg" alt="Data Engineering Process Fundamentals" style="width:140px; height:auto; border:0; margin-top:5px;" />
2018                </a></div></div>
2019</div><div class='widget AdSense' data-version='1' id='AdSense1'>
2020<div class='widget-content'>
2021<script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-3622730893924152&host=ca-host-pub-1556223355139109" crossorigin="anonymous"></script>
2021
2022<!-- ozkary_sidebarrightsec_AdSense1_1x1_as -->
2023<ins class="adsbygoogle"
2024     style="display:block"
2025     data-ad-client="ca-pub-3622730893924152"
2026     data-ad-host="ca-host-pub-1556223355139109"
2027     data-ad-slot="5298112308"
2028     data-ad-format="auto"
2029     data-full-width-responsive="true"></ins>
2030<script>
2031(adsbygoogle = window.adsbygoogle || []).push({});
2032</script>
2032
2033</div>
2034</div><div class='widget HTML' data-version='1' id='HTML4'>
2035<h2 class='title'>Developer Technologies Award</h2>
2036<div class='widget-content'><div class="responsive-video"><iframe src="https://www.youtube.com/embed/qlmP5_PdBTo?controls=0&start=3" title="YouTube video player" frameborder="0" allowfullscreen></iframe></div></div>
2037</div><div class='widget HTML' data-version='1' id='HTML3'>
2038<div class='widget-content'><aside class="sidebar-sections" style="font-family: system-ui, -apple-system, sans-serif;">
2039  <section style="margin-bottom: 25px;">
2040    <h3 style="font-size: 1.1rem; margin-bottom: 15px; border-bottom: 1px solid #e1e4e8; padding-bottom: 5px; color: #111;">Connect</h3>
2041    <ul style="list-style: none; padding: 0; margin: 0;">
2042      <li style="margin-bottom: 15px; display: flex; align-items: flex-start; gap: 10px;">
2043        <img src="https://cdn.jsdelivr.net/npm/simple-icons@v11/icons/github.svg" alt="" style="width: 20px; height: 20px; flex-shrink: 0; margin-top: 2px;" />
2044        <div>
2045          <a href="https://github.com/ozkary" target="_blank" rel="noopener" style="font-weight: bold; color: #0066cc; text-decoration: none;">GitHub</a>: 
2046          <span style="font-size: 0.85rem; color: #555;">Follow my open-source journey and star useful repos.</span>
2047        </div>
2048      </li>
2049      <li style="margin-bottom: 15px; display: flex; align-items: flex-start; gap: 10px;">
2050        <img src="https://cdn.jsdelivr.net/npm/simple-icons@v11/icons/youtube.svg" alt="" style="width: 20px; height: 20px; flex-shrink: 0; margin-top: 2px; filter: invert(11%) sepia(100%) saturate(6449%) hue-rotate(355deg) brightness(98%) contrast(118%);" />
2051        <div>
2052          <a href="https://www.youtube.com/@ozkary" target="_blank" rel="noopener" style="font-weight: bold; color: #0066cc; text-decoration: none;">YouTube</a>: 
2053          <span style="font-size: 0.85rem; color: #555;">Watch step-by-step engineering tutorials.</span>
2054        </div>
2055      </li>
2056      <li style="margin-bottom: 15px; display: flex; align-items: flex-start; gap: 10px;">
2057        <div style="display: flex; gap: 4px; flex-shrink: 0; margin-top: 2px;">
2058          <img src="https://cdn.jsdelivr.net/npm/simple-icons@v11/icons/bluesky.svg" alt="" style="width: 18px; height: 18px; filter: invert(41%) sepia(85%) saturate(2206%) hue-rotate(189deg) brightness(101%) contrast(105%);" />
2059          <img src="https://cdn.jsdelivr.net/npm/simple-icons@v11/icons/x.svg" alt="" style="width: 16px; height: 16px;" />
2060        </div>
2061        <div>
2062          <a href="https://bsky.app/profile/ozkary.bsky.social" target="_blank" rel="noopener" style="font-weight: bold; color: #0066cc; text-decoration: none;">BlueSky</a> 
2063          <span style="color: #ccc; margin: 0 2px;">|</span>
2064          <a href="https://x.com/ozkary" target="_blank" rel="noopener" style="font-weight: bold; color: #0066cc; text-decoration: none;">X</a>: 
2065          <span style="font-size: 0.85rem; color: #555;">Daily tech updates and quick tips.</span>
2066        </div>
2067      </li>
2068    </ul>
2069  </section>
2070
2071 <style>
2072  .sidebar-img-container {
2073      text-align: center !important;
2074    }
2075     .sidebar-responsive-img {
2076      max-width: 225px !important;
2077      margin: 0 auto !important;
2078      display: inline-block !important;
2079    }
2080  @media (max-width: 768px) { 
2081    .sidebar-responsive-img {
2082      max-width: 180px !important;
2083    }
2084  }
2085</style>
2086
2087<section style="margin-bottom: 25px;">
2088  <h3 style="font-size: 1.1rem; margin-bottom: 10px; border-bottom: 1px solid #e1e4e8; padding-bottom: 5px;">Recognitions</h3>
2089  <div class="sidebar-img-container" style="margin-bottom: 15px;">
2090    <img class="sidebar-responsive-img" src="https://www.ozkary.dev/assets/microsoft-mvp.jfif" alt="Microsoft MVP Badge" loading="lazy" style="max-width: 80%; height: auto;" />
2091  </div>
2092</section>
2093
2094<section style="margin-bottom: 25px;">
2095  <h3 style="font-size: 1.1rem; margin-bottom: 10px; border-bottom: 1px solid #e1e4e8; padding-bottom: 5px;">Communities</h3>
2096  <ul style="list-style: none; padding: 0; margin: 0;">
2097    <li style="margin-bottom: 15px;">
2098      <a href="https://gdg.community.dev/gdg-broward-county-fl/" target="_blank" rel="noopener" class="sidebar-img-container" style="display: block;">
2099        <img class="sidebar-responsive-img" src="https://www.ozkary.dev/assets/gdg-broward-logo.jpg" alt="GDG Broward County FL" loading="lazy" style="max-width: 80%; height: auto;" />
2100      </a>
2101    </li>
2102    <li style="margin-bottom: 15px;">
2103      <a href="https://globalai.community/chapters/jacksonville/" target="_blank" rel="noopener" class="sidebar-img-container" style="display: block;">
2104        <img class="sidebar-responsive-img" src="https://www.ozkary.dev/assets/global-ai-jacksonville-sm.png" alt="Global AI Jacksonville FL" loading="lazy" style="max-width: 225px; height: auto;" />
2105      </a>
2106    </li>
2107  </ul>
2108</section>
2109</aside></div>
2110</div><div class='widget HTML' data-version='1' id='HTML2'>
2111<h2 class='title'>Protected Content</h2>
2112<div class='widget-content'><a href="//www.dmca.com/Protection/Status.aspx?ID=de6be0cf-25cd-4a7f-97b2-852981ecccc6" target="_blank" title="Check blog Protection Status" class="dmca-badge"> <img src ="//www.dmca.com/Badges/dmca_protected_sml_120n.png" alt="DMCA.com for Blogger blogs" /></a> 
2113<script src="//images.dmca.com/Badges/DMCABadgeHelper.min.js"></script>
vendor: 1 bytes, line 2113
2113
2114<script async="async" data-uid="2b8578a74a" src="https://ozkary.kit.com/2b8578a74a/index.js"></script>
2114</div>
2115</div></div>
2116</aside>
2117</div>
2118<footer class='site-footerpbt' id='colophonpbt' role='contentinfo' style='clear: both; text-align: center; padding: 20px 0; font-size: 0.9rem; color: #6a737d;'>
2119<div class='site-infopbt'>
2120          Copyright &copy; 2009 - <span id='current-year'></span>
2121<a href='https://www.ozkary.com/'>Ozkary -  Emerging Technologies</a> | 
2122          <a href='//www.ozkary.com/p/privacy.html'>Privacy</a>
2123<br/>Use the content of this blog as a reference only.
2124        </div>
2125</footer>
2126</div>
2127<script src='https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js' type='text/javascript'></script>
vendor: 1 bytes, line 2127
2127
2128<script crossorigin='anonymous' src='https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js'></script>
vendor: 1 bytes, line 2128
2128
2129<script src='//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.7.0/highlight.min.js'></script>
vendor: 1 bytes, line 2129
2129
2130<script type='text/javascript'>
2131      //<![CDATA[
2132      document.addEventListener('DOMContentLoaded', function() {
2133        // Dynamic Footer Year
2134        document.getElementById('current-year').textContent = new Date().getFullYear();
2135
2136        // Initialize Syntax Highlighting
2137        hljs.highlightAll();
2138
2139        // Add Clipboard "Copy" Button to Pre/Code Blocks
2140        document.querySelectorAll('pre').forEach(function(codeBlock) {
2141          var copyBtn = document.createElement('button');
2142          copyBtn.className = 'copy-code-btn btn btn-sm btn-outline-light';
2143          copyBtn.innerText = 'Copy';
2144          copyBtn.style.cssText = 'position: absolute; top: 8px; right: 8px; font-size: 11px; padding: 2px 8px; opacity: 0.8; z-index: 10;';
2145          
2146          copyBtn.addEventListener('click', function() {
2147            var codeText = codeBlock.querySelector('code') ? codeBlock.querySelector('code').innerText : codeBlock.innerText;
2148            navigator.clipboard.writeText(codeText).then(function() {
2149              copyBtn.innerText = 'Copied!';
2150              setTimeout(function() { copyBtn.innerText = 'Copy'; }, 2000);
2151            });
2152          });
2153          
2154          codeBlock.style.position = 'relative';
2155          codeBlock.appendChild(copyBtn);
2156        });
2157
2158        // Calculate reading time for post headers
2159        document.querySelectorAll('.post-read-time').forEach(function(el) {
2160          var postId = el.getAttribute('data-post-id');
2161          var postBody = document.getElementById('post-body-full-' + postId) || document.getElementById('post-body-' + postId);
2162          if (postBody) {
2163            var text = postBody.textContent || postBody.innerText || '';
2164            var words = text.trim().split(/\s+/).length;
2165            var minRead = Math.max(1, Math.ceil(words / 200));
2166            el.textContent = minRead + ' min read';
2167          } else {
2168            el.style.display = 'none';
2169          }
2170        });
2171      });
2172      //]]>
2173    </script>
2173
2174<noscript><iframe height='0' src='https://www.googletagmanager.com/ns.html?id=GTM-58XJDSH' style='display:none;visibility:hidden' width='0'></iframe></noscript>
2175
2176<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/851759228-widgets.js"></script>
vendor: 1 bytes, line 2176
2176
2177<script type='text/javascript'>
2178window['__wavt'] = 'AAPvtVnUbLp9C4kHWLnulau4llzv:1791055518205';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d2250750882473644809','//www.ozkary.com/search?updated-max\x3d2026-03-31T17:00:00-07:00\x26max-results\x3d7','2250750882473644809');
2179_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '2250750882473644809', 'title': 'Ozkary -  Emerging Technologies', 'url': 'https://www.ozkary.com/search?updated-max\x3d2026-03-31T17:00:00-07:00\x26max-results\x3d7', 'canonicalUrl': 'https://www.ozkary.com/search?updated-max\x3d2026-03-31T17:00:00-07:00\x26max-results\x3d7', 'homepageUrl': 'https://www.ozkary.com/', 'searchUrl': 'https://www.ozkary.com/search', 'canonicalHomepageUrl': 'https://www.ozkary.com/', 'blogspotFaviconUrl': 'https://www.ozkary.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': true, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'UA-64364631-1', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Ozkary -  Emerging Technologies - Atom\x22 href\x3d\x22https://www.ozkary.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22Ozkary -  Emerging Technologies - RSS\x22 href\x3d\x22https://www.ozkary.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22Ozkary -  Emerging Technologies - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/2250750882473644809/posts/default\x22 /\x3e\n', 'meTag': '\x3clink rel\x3d\x22me\x22 href\x3d\x22https://www.blogger.com/profile/13427831719934915504\x22 /\x3e\n', 'adsenseClientId': 'ca-pub-3622730893924152', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': true, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'isGoogleEverywhereLinkTooltipEnabled': true, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/7ce469edaf870342', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}
2179, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'index', 'pageName': '', 'pageTitle': 'Ozkary -  Emerging Technologies', 'metaDescription': 'Software development, JavaScript, .NET, ASP.NET, Visual Studio, database, Azure, react, angular, NodeJS, chatbot, SQL, Data Engineering Fundamentals'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Ozkary -  Emerging Technologies', 'description': 'Software development, JavaScript, .NET, ASP.NET, Visual Studio, database, Azure, react, angular, NodeJS, chatbot, SQL, Data Engineering Fundamentals', 'url': 'https://www.ozkary.com/search?updated-max\x3d2026-03-31T17:00:00-07:00\x26max-results\x3d7', 'type': 'feed', 'isSingleItem': false, 'isMultipleItems': true, 'isError': false, 'isPage': false, 'isPost': false, 'isHomepage': false, 'isArchive': false, 'isSearch': true, 'isLabelSearch': false, 'search': {}}}]);
2180_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'headersec', document.getElementById('Header1'), {}, 'displayModeFull'));
2181_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'mainblogsec', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/120969204-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/828616780-lightbox_bundle.css'}, 'displayModeFull'));
2182_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML1', 'sidebarrightsec', document.getElementById('HTML1'), {}, 'displayModeFull'));
2183_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML5', 'sidebarrightsec', document.getElementById('HTML5'), {}, 'displayModeFull'));
2184_WidgetManager._RegisterWidget('_AdSenseView', new _WidgetInfo('AdSense1', 'sidebarrightsec', document.getElementById('AdSense1'), {}, 'displayModeFull'));
2185_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML4', 'sidebarrightsec', document.getElementById('HTML4'), {}, 'displayModeFull'));
2186_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML3', 'sidebarrightsec', document.getElementById('HTML3'), {}, 'displayModeFull'));
2187_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'sidebarrightsec', document.getElementById('HTML2'), {}, 'displayModeFull'));
2188</script>
2188
2189</body>
2190</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.