PageSourceSearch

https://mach5.io/docs/6.2.0/ingestion/kafka

html mach5.io collected 2026-09-24 09:43:48 UTC 63,858 bytes, 241 lines download raw bytes

1<!DOCTYPE html><html lang="en"><head><link rel="icon" type="image/x-icon" href="/images/favicon.ico"><meta charset="utf-8"><!--HEAD--><link id="leptos" rel="stylesheet" href="/pkg/website.css"><meta name="description" content="Connect Kafka topics to Mach5 Search. Set schemas and reliable streaming"><title>Ingesting Kafka Data | Mach5 Technical Documentation</title><meta name="viewport" content="width=device-width, initial-scale=1"><!><link rel="modulepreload" href="/pkg/website.js" nonce="9ak8jQ8_yXW6RprR1oxjTw"><link rel="preload" href="/pkg/website.wasm" as="fetch" type="application/wasm" crossorigin="9ak8jQ8_yXW6RprR1oxjTw">
1<script type="module" nonce="9ak8jQ8_yXW6RprR1oxjTw">((root, pkg_path, output_name, wasm_output_name) => {
2	let MOST_RECENT_CHILDREN_CB = [];
3
4	function idle(c) {
5		if ("requestIdleCallback" in window) {
6			window.requestIdleCallback(c);
7		} else {
8			c();
9		}
10	}
11	function hydrateIslands(rootNode, mod) {
12		function traverse(node) {
13			if (node.nodeType === Node.ELEMENT_NODE) {
14				const tag = node.tagName.toLowerCase();
15				if(tag === 'leptos-island') {
16					const children = [];
17					const id = node.dataset.component || null;
18
19					hydrateIsland(node, id, mod);
20					
21					for(const child of node.children) {
22						traverse(child, children);
23					}
24				} else {
25					if (tag === 'leptos-children') {
26						MOST_RECENT_CHILDREN_CB.push(node.$$on_hydrate);
27						for(const child of node.children) {
28							traverse(child);
29						};
30						// un-set the "most recent children"
31						MOST_RECENT_CHILDREN_CB.pop();
32					} else {
33						for(const child of node.children) {
34							traverse(child);
35						};
36					}
37				}
38			}
39		}
40
41		traverse(rootNode);
42	}
43	function hydrateIsland(el, id, mod) {
44		const islandFn = mod[id];
45		if (islandFn) {
46			const children_cb = MOST_RECENT_CHILDREN_CB[MOST_RECENT_CHILDREN_CB.length-1];
47			if (children_cb) {
48				children_cb();
49			}
50			islandFn(el);
51		} else {
52			console.warn(`Could not find WASM function for the island ${id}.`);
53		}
54	}
55	idle(() => {
56		import(`${root}/${pkg_path}/${output_name}.js`)
57			.then(mod => {
58				mod.default(`${root}/${pkg_path}/${wasm_output_name}.wasm`).then(() => {
59					mod.hydrate();
60					hydrateIslands(document.body, mod);
61				});
62
63				window.__hydrateIsland = (el, id) => hydrateIsland(el, id, mod);
64			})
65	});
66})
67("", "pkg", "website", "website");</script>
67<script src="/js/site.js?v=20260418-cookie-banner" defer></script>
67</head><body class="bg-[#F5F1EA] text-[#18222B] overflow-x-hidden"><nav class="fixed top-0 left-0 right-0 z-50 border-b border-[#31556F]/15 bg-[#F5F1EA]/95 backdrop-blur"><div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8"><div class="h-16 flex items-center justify-between gap-6"><a href="/" class="flex items-center gap-3 text-[#18222B]"><img src="/images/mach5.svg" alt="Mach5" class="h-8 w-8 rounded-md"><span class="font-space-grotesk text-xl font-semibold">Mach5</span></a><div class="hidden lg:flex items-center gap-1 text-sm font-medium text-[#1F2C36]"><div class="relative group"><button class="flex items-center gap-1 px-3 py-2 rounded-md hover:text-[#FF5C1B] hover:bg-[#FFFDF9] transition-colors"><span>Platform</span><svg fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-3.5 h-3.5"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2.5" d="M19 9l-7 7-7-7"></path></svg></button><div class="absolute left-0 top-full pt-3 hidden group-hover:block"><div class="w-[520px] rounded-xl border border-[#31556F]/15 bg-[#FFFDF9] shadow-xl p-5"><div class="grid grid-cols-1 gap-1"><a href="/platform" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Platform Overview</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Search, analytics, streaming, derived data, and product-serving security infrastructure.</div></a><a href="/platform/search-analytics" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Search &amp; Analytics</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Low-latency customer-facing security search and analytics workloads.</div></a><a href="/platform/streaming" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Streaming Infrastructure</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Ingest, transform, enrich, backfill, and serve fresh derived security data.</div></a><a href="/platform/integrations" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Integrations</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Connect streams, object stores, warehouses, SaaS systems, and product workflows.</div></a></div></div></div></div><div class="relative group"><button class="flex items-center gap-1 px-3 py-2 rounded-md hover:text-[#FF5C1B] hover:bg-[#FFFDF9] transition-colors"><span>Workloads</span><svg fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-3.5 h-3.5"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2.5" d="M19 9l-7 7-7-7"></path></svg></button><div class="absolute left-0 top-full pt-3 hidden group-hover:block"><div class="w-[560px] rounded-xl border border-[#31556F]/15 bg-[#FFFDF9] shadow-xl p-5"><div class="grid grid-cols-1 gap-1"><a href="/solutions" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Workloads Hub</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Start with one painful search, analytics, streaming, or pipeline workload.</div></a><a href="/solutions/real-time-search" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Customer-Facing Search</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Fast product search without operating fragile search clusters.</div></a><a href="/resources/migrate-from-elasticsearch" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Elasticsearch / OpenSearch Relief</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Reduce cost and operational burden while preserving search experiences.</div></a><a href="/solutions/log-analysis" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Multi-Tenant Analytics</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Tenant-aware security analytics and dashboards for product teams.</div></a><a href="/docs/6.2.0/ingestion/kafka" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Streaming Pipelines</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Ingest, transform, enrich, and backfill high-volume security event streams.</div></a></div></div></div></div><div class="relative group"><button class="flex items-center gap-1 px-3 py-2 rounded-md hover:text-[#FF5C1B] hover:bg-[#FFFDF9] transition-colors"><span>Customers</span><svg fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-3.5 h-3.5"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2.5" d="M19 9l-7 7-7-7"></path></svg></button><div class="absolute left-0 top-full pt-3 hidden group-hover:block"><div class="w-[440px] rounded-xl border border-[#31556F]/15 bg-[#FFFDF9] shadow-xl p-5"><div class="grid grid-cols-1 gap-1"><a href="/case-studies" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Customer Stories</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">How security companies use Mach5 in production.</div></a><a href="/case-studies/permiso" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Permiso Case Study</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">
67From low-latency UI search to broader backend data infrastructure.</div></a><a href="/incidentbench" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Benchmarks</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Performance and infrastructure proof for high-volume analytics.</div></a></div></div></div></div><div class="relative group"><button class="flex items-center gap-1 px-3 py-2 rounded-md hover:text-[#FF5C1B] hover:bg-[#FFFDF9] transition-colors"><span>Resources</span><svg fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-3.5 h-3.5"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2.5" d="M19 9l-7 7-7-7"></path></svg></button><div class="absolute right-0 top-full pt-3 hidden group-hover:block"><div class="w-[500px] rounded-xl border border-[#31556F]/15 bg-[#FFFDF9] shadow-xl p-5"><div class="grid grid-cols-1 gap-1"><a href="/resources" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Resource Library</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Architecture, migration, benchmarks, and security infrastructure writing.</div></a><a href="/resources/migrate-from-elasticsearch" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Migrate from Elasticsearch</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Search cost, scale, and operational burden relief.</div></a><a href="/resources/mach5-building-a-low-latency-search-engine-object-storage" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Architecture Deep Dive</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">How Mach5 approaches low-latency search on object storage.</div></a><a href="/company" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Company</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">The team building Mach5 security data infrastructure.</div></a><a href="/company/contact" class="block rounded-lg px-3 py-2.5 hover:bg-[#F5F1EA] transition-colors"><div class="text-sm font-semibold text-[#18222B] hover:text-[#FF5C1B]">Contact</div><div class="mt-0.5 text-xs text-[#52606C] leading-snug">Talk to the Mach5 team.</div></a></div></div></div></div></div><div class="hidden sm:flex items-center gap-3"><a href="/docs/6.2.0" class="text-sm font-semibold text-[#31556F] hover:text-[#52748D] transition-colors">Docs</a><a href="/schedule-a-demo" class="rounded-lg bg-[#FF5C1B] px-4 py-2 text-sm font-semibold text-white hover:bg-[#F76D34] transition-colors">Book a Demo</a></div><div class="lg:hidden flex items-center gap-3"><button type="button" onclick="toggleMenu()" aria-label="Open menu" class="text-[#31556F] hover:text-[#18222B] focus:outline-none"><svg class="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16M4 18h16"></path></svg></button></div></div><div id="mobile-menu" class="lg:hidden hidden pb-4 pt-2 space-y-1 text-sm text-[#1F2C36]"><div class="border-b border-[#31556F]/10 last:border-b-0"><button type="button" onclick="toggleDropdown('mobile-platform')" class="flex items-center justify-between w-full px-3 py-3 text-left font-semibold text-[#18222B]"><span>Platform</span><svg id="mobile-platform-arrow" fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-4 h-4 text-[#31556F] transition-transform duration-200"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"></path></svg></button><div id="mobile-platform" class="hidden pb-2 pl-3"><a href="/platform" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Platform Overview</a><a href="/platform/search-analytics" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Search &amp; Analytics</a><a href="/platform/streaming" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Streaming</a><a href="/platform/integrations" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Integrations</a><!></div></div><div class="border-b border-[#31556F]/10 last:border-b-0"><button type="button" onclick="toggleDropdown('mobile-workloads')" class="flex items-center justify-between w-full px-3 py-3 text-left font-semibold text-[#18222B]"><span>Workloads</span><svg id="mobile-workloads-arrow" fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-4 h-4 text-[#31556F] transition-transform duration-200"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"></path></svg></button><div id="mobile-workloads" class="hidden pb-2 pl-3"><a href="/solutions" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Workloads Hub</a><a href="/solutions/real-time-search" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Customer-Facing Search</a><a href="/resources/migrate-from-elasticsearch" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Migrate from Elasticsearch</a><a href="/docs/6.2.0/ingestion/kafka" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Streaming Pipelines</a><!></div></div><div class="border-b border-[#31556F]/10 last:border-b-0"><button type="button" onclick="toggleDropdown('mobile-customers')" class="flex items-center justify-between w-full px-3 py-3 text-left font-semibold text-[#18222B]"><span>Customers</span><svg id="mobile-customers-arrow" fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-4 h-4 text-[#31556F] transition-transform duration-200"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"></path></svg></button><div id="mobile-customers" class="hidden pb-2 pl-3"><a href="/case-studies" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Customer Stories</a><a href="/case-studies/permiso" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Permiso</a><a href="/incidentbench" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">
67Benchmarks</a><!></div></div><div class="border-b border-[#31556F]/10 last:border-b-0"><button type="button" onclick="toggleDropdown('mobile-resources')" class="flex items-center justify-between w-full px-3 py-3 text-left font-semibold text-[#18222B]"><span>Resources</span><svg id="mobile-resources-arrow" fill="none" stroke="currentColor" viewBox="0 0 24 24" class="w-4 h-4 text-[#31556F] transition-transform duration-200"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"></path></svg></button><div id="mobile-resources" class="hidden pb-2 pl-3"><a href="/resources" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Resource Library</a><a href="/docs/6.2.0" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Docs</a><a href="/company" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Company</a><a href="/company/contact" class="block px-3 py-2 text-sm text-[#1F2C36] hover:text-[#FF5C1B]">Contact</a><!></div></div><a href="/schedule-a-demo" class="block mt-3 rounded-lg bg-[#FF5C1B] px-4 py-2.5 text-center text-sm font-semibold text-white hover:bg-[#F76D34] transition-colors">Book a Demo</a></div></div></nav><main><style>
68            /* Webkit scrollbar styling for Chrome */
69            .scrollbar-thin::-webkit-scrollbar {
70                width: 6px;
71                height: 6px;
72            }
73            .scrollbar-thin::-webkit-scrollbar-track {
74                background: transparent;
75            }
76            .scrollbar-thin::-webkit-scrollbar-thumb {
77                background-color: transparent;
78                border-radius: 10px;
79                border: 2px solid transparent;
80                transition: background-color 0.3s ease;
81            }
82            .scrollbar-thin:hover::-webkit-scrollbar-thumb {
83                background-color: rgba(107, 114, 128, 0.5); /* Tailwind gray-500 with opacity */
84            }
85            </style><div class="h-screen flex flex-col bg-[#F5F1EA] text-[#18222B]"><main class="flex flex-1 overflow-hidden px-2 mb-2"><div id="docs-nav-sidebar" class="w-[16.5rem] flex-shrink-0 overflow-y-auto scrollbar-thin scrollbar-thumb-transparent hover:scrollbar-thumb-gray-500 scrollbar-track-transparent hidden md:block bg-[#FFFDF9] border-r border-[#31556F]/15"><div class="w-64 flex-shrink-0 hidden md:block mt-20"><div class="sticky top-8"><section class="border-gray-300 rounded-lg p-4 shadow-sm"><div class="left-column"><div class="pl-4 pr-2 pb-4 mb-3 border-b border-[#31556F]/15"><div class="text-xs font-semibold uppercase tracking-wide text-[#52606C]">Mach5 Search docs</div><div class="mt-2 flex items-center justify-between rounded-md bg-[#F7F3ED] px-3 py-2"><span class="text-sm font-semibold text-[#18222B]">Version <!>6.2.0</span><span class="rounded-full bg-[#FFE8DC] px-2 py-0.5 text-xs font-semibold text-[#C2410C]">latest</span></div><a href="/release-notes/v6-2-0" class="mt-2 block text-xs font-medium text-[#FF5C1B] hover:underline">Release notes for 6.2.0</a><div class="mt-4"><div class="mb-1 text-xs font-semibold uppercase tracking-wide text-[#52606C]">Versions</div><div class="space-y-1"><a href="/docs/6.2.0" class="block rounded-md bg-[#FFE8DC] px-3 py-2 text-sm font-semibold text-[#18222B]">6.2.0 <span class="text-xs text-[#52606C]">latest</span></a><a href="/docs/6.1.0" class="block rounded-md px-3 py-2 text-sm font-medium text-[#52606C] hover:bg-[#F7F3ED] hover:text-[#18222B]">6.1.0</a></div></div></div><ul class="w-full"><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Mach5 One</h3><li class="w-full"><a href="/docs/6.2.0/one" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Overview</a></li><li class="w-full"><a href="/docs/6.2.0/one/getting-started" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Getting Started</a></li><li class="w-full"><a href="/docs/6.2.0/one/install" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Install</a></li><li class="w-full"><a href="/docs/6.2.0/one/docker-setup" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Docker Setup</a></li><li class="w-full"><a href="/docs/6.2.0/one/quickstart" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Quickstart</a></li><li class="w-full"><a href="/docs/6.2.0/one/configuration" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Configuration</a></li><li class="w-full"><a href="/docs/6.2.0/one/ingest-examples" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Ingest Examples</a></li><li class="w-full"><a href="/docs/6.2.0/one/query-examples" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Query Examples</a></li><li class="w-full"><a href="/docs/6.2.0/one/mcp" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">MCP</a></li><li class="w-full"><a href="/docs/6.2.0/one/agent-quickstart" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Agent Quickstart</a></li><li class="w-full"><a href="/docs/6.2.0/one/agents" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Agent Instructions</a></li><li class="w-full"><a href="/docs/6.2.0/one/where-to-find-apps" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Where to Find Apps</a></li><li class="w-full"><a href="/docs/6.2.0/one/troubleshooting" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Troubleshooting</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">m5c</h3><li class="w-full"><a href="/docs/6.2.0/m5c" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Overview</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/workspaces-packages" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Workspaces &amp; Packages</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/data-contracts" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Data Contracts</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/data-modules" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Data Modules</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/contract-mappings" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Contract Mappings</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/detection-families" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Detection Families</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/detections" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Detections</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/detection-imports" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Detection Imports</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/app-bundles" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">
85App Bundles</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/mcp" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">MCP Server</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/smoke-tests" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Smoke Tests</a></li><li class="w-full"><a href="/docs/6.2.0/m5c/commands" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Command Reference</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Overview</h3><li class="w-full"><a href="/docs/6.2.0/overview/architecture" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Architecture</a></li><li class="w-full"><a href="/docs/6.2.0/overview/docs-scope" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Docs Scope</a></li><li class="w-full"><a href="/docs/6.2.0/overview/one-vs-enterprise" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">One vs Enterprise</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Getting Started</h3><li class="w-full"><a href="/docs/6.2.0/getting-started/quickstart" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Quickstart Guide</a></li><li class="w-full"><a href="/docs/6.2.0/getting-started/stores-and-store-routes" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Stores and Store Routes</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Mach5 Enterprise</h3><li class="w-full"><a href="/docs/6.2.0/deploy/aws-marketplace" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">AWS Marketplace</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/aks-cluster-configuration" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">AKS Cluster Configuration</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/initialize-aks" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Initialize Mach5 Search in AKS</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/eks-cluster-configuration" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">EKS Cluster Configuration</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/gke-cluster-configuration" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">GKE Cluster Configuration</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/initialize-gke" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Initialize Mach5 Search in GKE</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/helm-charts" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Helm Charts</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/local-kubernetes" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Local Kubernetes</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/license-token" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">License Token</a></li><li class="w-full"><a href="/docs/6.2.0/deploy/node-to-pod-mapping" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Node to Pod Mapping</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Both Editions</h3><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Ingestion</h3><li class="w-full"><a href="/docs/6.2.0/ingestion/transform-functions" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Transform Functions</a></li><li class="w-full"><a href="/docs/6.2.0/ingestion/kafka" class="docs-nav-active block w-full pl-4 py-2 text-sm font-semibold transition-colors border-l-2 bg-[#FFE8DC] text-[#18222B] border-[#FF5C1B]">Kafka</a></li><li class="w-full"><a href="/docs/6.2.0/ingestion/s3" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">S3</a></li><li class="w-full"><a href="/docs/6.2.0/ingestion/iceberg" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Iceberg</a></li><li class="w-full"><a href="/docs/6.2.0/ingestion/operation-modes" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Operation Modes</a></li><li class="w-full"><a href="/docs/6.2.0/ingestion/synthetic-data-generation" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Synthetic Data Generation</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Querying</h3><li class="w-full"><a href="/docs/6.2.0/querying/sql" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">SQL Support</a></li><li class="w-full"><a href="/docs/6.2.0/querying/materialized-views" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Materialized Views</a></li><li class="w-full"><a href="/docs/6.2.0/querying/materialized-views-legacy" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Materialized Views v1 (Deprecated)</a></li><li class="w-full"><a href="/docs/6.2.0/querying/kql" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">KQL Support</a></li><li class="w-full"><a href="/docs/6.2.0/querying/kql-client" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">KQL Client</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Security</h3><li class="w-full"><a href="/docs/6.2.0/security/authentication-authorization" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Authentication and Authorization</a></li><li class="w-full"><a href="/docs/6.2.0/security/keycloak-admin" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Keycloak Admin</a></li><li class="w-full"><a href="/docs/6.2.0/security/role-patterns" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Role Patterns</a></li><li class="w-full"><a href="/docs/6.2.0/security/authenticated-api-access" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Authenticated API Access</a></li><li class="w-full"><a href="/docs/6.2.0/security/admin-role" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Admin Role</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Enterprise Operations</h3><li class="w-full"><a href="/docs/6.2.0/operate/index-metadata-prefetch" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Index Metadata Prefetch</a></li><li class="w-full"><a href="/docs/6.2.0/operate/observability" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Observability</a></li><li class="w-full"><a href="/docs/6.2.0/operate/telemetry" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Telemetry</a></li><!><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Workflows</h3><li class="w-full"><a href="/docs/6.2.0/workflows/axon/concepts" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Axon Concepts</a></li><li class="w-full"><a href="/docs/6.2.0/workflows/axon/syntax" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Axon Syntax</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Declarative Apps</h3><li class="w-full"><a href="/docs/6.2.0/apps/declarative" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">App Concepts</a></li><li class="w-full"><a href="/docs/6.2.0/apps/declarative/examples" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">App Examples</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">
85App Bundles</h3><li class="w-full"><a href="/docs/6.2.0/apps/bundles" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Bundle Concepts</a></li><li class="w-full"><a href="/docs/6.2.0/apps/bundles/examples" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Bundle Examples</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Notebooks</h3><li class="w-full"><a href="/docs/6.2.0/notebooks/concepts" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Notebook Concepts</a></li><li class="w-full"><a href="/docs/6.2.0/notebooks/examples" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Notebook Examples</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Dashboards</h3><li class="w-full"><a href="/docs/6.2.0/dashboards/sample-data" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Sample Data and Visualizations</a></li><li class="w-full"><a href="/docs/6.2.0/dashboards/concepts" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Dashboard Concepts</a></li><li class="w-full"><a href="/docs/6.2.0/dashboards/configuration" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Dashboard Configuration</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Integrations</h3><li class="w-full"><a href="/docs/6.2.0/integrations/github" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">GitHub</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/slack" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Slack</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/okta" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Okta</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/amazon-s3" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Amazon S3</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/google-cloud-storage" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Google Cloud Storage</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/azure-blob-storage" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Azure Blob Storage</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/imap" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">IMAP</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/pop3" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">POP3</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/smtp" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">SMTP</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/databricks" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Databricks</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/trino-opensearch-connector" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Trino OpenSearch Connector</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/cloudflare-data-platform" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Cloudflare Data Platform</a></li><li class="w-full"><a href="/docs/6.2.0/integrations/rook-ceph-guide" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Rook-Ceph</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Reference</h3><li class="w-full"><a href="/docs/6.2.0/reference/opensearch-apis" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">OpenSearch APIs</a></li><h3 class="text-base font-bold mt-5 mb-1 text-orange-600 pl-4">Support</h3><li class="w-full"><a href="/docs/6.2.0/support" class="block w-full pl-4 py-2 text-sm font-medium transition-colors border-l-2 text-[#52606C] bg-transparent border-transparent hover:bg-[#F7F3ED] hover:text-[#18222B]">Support</a></li></ul></div></section></div></div></div><div class="flex-1 min-w-0 overflow-y-auto scrollbar-thin scrollbar-thumb-transparent hover:scrollbar-thumb-gray-500 scrollbar-track-transparent mx-2 border-r border-[#31556F]/15 bg-[#F5F1EA]"><div class="flex-1 min-w-0 overflow-hidden break-words mt-10"><link rel="stylesheet" href="/styles/github-markdown.min.css"><link rel="stylesheet" href="/styles/github.min.css"><link rel="stylesheet" href="/styles/override-github-markdown.min.css"><div class="markdown-body w-full px-4 py-6"><!><article><h1 id="ingesting-data-from-kafka-into-mach5" class="scroll-target">Ingesting data from Kafka into Mach5</h1>
86<p>This document explains how to ingest data from Kafka topics into a Mach5 Search index by configuring connections, setting up ingest pipelines, and verifying ingestion using the Mach5 UI.</p>
87<h2 id="prerequisites" class="scroll-target">Prerequisites</h2>
88<ul>
89<li>This document assumes that Mach5 is deployed and running successfully. Mach5 Administrative UI page looks as below. Lets assume it’s running at <a href="http://localhost:8888/">http://localhost:8888/</a></li>
90</ul>
91<img src="/images/docs/common/firstpageui-noversion-adminuser.png" alt="FirstPageUI" style="max-width: 100%; height: auto;" />
92<ul>
93<li>Store, store route and warehouse are created successfully. Refer to <a href="https://mach5.io/docs/quickstart">Quickstart</a> document for help</li>
94<li>Consider there is a Kafka topic named <strong>kafkatopic</strong> already set up having following 10 records:</li>
95</ul>
96<pre><code>@kafka-client$ kafka-console-consumer.sh --bootstrap-server kafk
96a.kafka.svc.cluster.local:9092 --topic kafkatopic --from-beginning
97{ "id": "1", "ip_address": "192.168.1.1" } 
98{ "id": "2", "ip_address": ["192.168.1.2", "10.0.0.5"] } 
99{ "id": "3", "ip_address": "10.0.0.1" }
100{ "id": "4", "ip_address": ["10.0.0.2", "8.8.8.8"] } 
101{ "id": "5", "ip_address": "172.16.0.1" }
102{ "id": "6", "ip_address": ["172.16.0.2", "192.168.1.1"] } 
103{ "id": "7", "ip_address": "8.8.8.8" }
104{ "id": "8", "ip_address": ["8.8.4.4", "10.0.0.5"] }
105{ "id": "9", "ip_address": "192.168.2.1" } 
106{ "id": "10", "ip_address": ["192.168.2.2", "8.8.4.4"] }
107Processed a total of 10 messages
108</code></pre>
109<ul>
110<li>Mach5 Index is already created with the name <strong>kafkaindex</strong> and with relevant mappings for the kafka topic. If no mappings are provided, Mach5 will dynamically infer the mapping when ingesting data. <em>Please note that some data may not get ingested if mappings are not provided.</em></li>
111</ul>
112<h2 id="connections" class="scroll-target">Connections</h2>
113<p>Connection is the resource or the entity which stores the properties needed to connect or access the source where the data resides, for instance in Iceberg, Kafka or S3. It is important to note that the same connection can be used in multiple ingest pipelines</p>
114<p>Click on <strong>Connections</strong> on the left panel of Mach5 UI</p>
115<h3 id="add-a-new-connection" class="scroll-target">Add a new connection</h3>
116<p>Click on + icon on Connections page to create a new connection</p>
117<img src="/images/docs/ingestingkafkadata/kafka-connection-creation.png" alt="KafkaConnectionCreation" style="max-width: 58%; height: auto;" />
118<ul>
119<li><strong>Name:</strong> Provide a name for the connection, eg. <strong>kafka-conn</strong></li>
120<li><strong>ConnectionType:</strong> There are 3 options in dropdown: AwsS3, Iceberg, Kafka. Choose <strong>Kafka</strong></li>
121<li><strong>Bootstrap Servers</strong>: Specify the bootstrap server details. Eg: <strong>kafka.kafka.svc.cluster.local:9092</strong></li>
122<li><strong>Authentication</strong>: Keep it as default - <strong>None</strong>. Other option is AWS_MSK_IAM
123<ul>
124<li><strong>Enable Protocol SSL:</strong> Leave it <strong>deselected</strong> to use Kafka without SSL</li>
125<li><strong>Enable Verify Certificate:</strong> Leave it <strong>deselected</strong></li>
126</ul>
127</li>
128<li><strong>Test:</strong> Click on the Test button to verify connectivity to Kafka</li>
129<li>Click on <strong>Save</strong></li>
130</ul>
131<img src="/images/docs/ingestingkafkadata/kafka-connection-test.png" alt="KafkaConnectionTest" style="max-width: 58%; height: auto;" />
132<h3 id="verify-new-connection" class="scroll-target">Verify new connection</h3>
133<p>Verify that the new connection is created in the Connections page</p>
134<img src="/images/docs/ingestingkafkadata/kafka-connection-details.png" alt="KafkaConnectionDetails" style="max-width: 48%; height: auto;" />
135<h2 id="ingest-pipelines" class="scroll-target">Ingest Pipelines</h2>
136<p>Mach5 Search ingest pipelines allow you to process and ingest documents from various different sources like Iceberg, Kafka. S3 bucket, etc. This is useful for transforming, enriching, or modifying data at ingestion time. To access the source data, ingest pipeline needs the specific connection to source data. For example if we are creating an ingest pipeline to index data from Kafka topic as data source, then you need a connection of type Kafka</p>
137<p>Click on <strong>Ingest Pipelines</strong> on the left panel of Mach5 UI</p>
138<h3 id="add-an-ingest-pipeline" class="scroll-target">Add an ingest pipeline</h3>
139<p>Click on + icon on Ingest Pipelines page to create a new ingest pipeline</p>
140<img src="/images/docs/ingestingkafkadata/kafka-ip-creation1.png" alt="IngestPipelineCreation1" style="max-width: 58%; height: auto;" />
141<ul>
142<li><strong>Name:</strong> Provide name of the ingest pipeline, eg. <strong>kafka-ip</strong></li>
143<li><strong>Index:</strong> Specify name of the Mach5 index eg. <strong>kafkaindex</strong>. Please note that the index must be created in Mach5 prior to configuring the ingest pipeline</li>
144<li><strong>Transform Type</strong>: Select options between None or Javascript. This helps to transform data before ingestion. If Javascript is selected, specify the script details in the given box</li>
145<li><strong>Connection Name</strong>: Select the Kafka connection name that was created earlier</li>
146</ul>
147<img src="/images/docs/ingestingkafkadata/kafka-ip-creation2.png" alt="IngestPipelineCreation2" style="max-width: 58%; height: auto;" />
148<ul>
149<li><strong>Ingest Pipeline Type:</strong> Options are S3, Iceberg, Kafka. Select <strong>Kafka</strong></li>
150<li><strong>Topic</strong>: Specify the Kafka topic from which data needs to be ingested. Eg. <strong>kafkatopic</strong></li>
151<li><strong>Start Offset:</strong> Options are Earliest, Latest. Keep default as <strong>Earliest</strong>. Earliest means from the beginning of the topic. Latest means whatever data comes in the topic after the pipeline is setup</li>
152<li><strong>Group Id:</strong> Specify the group Id, eg. <strong>gidkafka</strong>. The group id is the unique identifier for a  Kafka consumer group</li>
153<li><strong>Offsets Per Batch</strong>: Default is 16384. Specify the offset as needed, otherwise keep as default</li>
154<li><strong>Advanced:</strong> Leave the advanced section as default, Operation Mode being Append/Upsert</li>
155<li><strong>Enabled</strong>: Select this checkbox to Enable the Ingest Pipeline</li>
156<li>Click on <strong>Save</strong></li>
157<li>Once the ingest pipeline is created, records will start getting reflected in the Mach5 index corresponding to the Kafka topic.</li>
158</ul>
159<h3 id="verify-an-ingest-pipeline" class="scroll-target">Verify an ingest pipeline</h3>
160<p>In Ingest Pipelines page verify if the ingestion pipeline is created properly</p>
161<img src="/images/docs/ingestingkafkadata/kafka-ip-details.png" alt="IngestPipelineDetails" style="max-width: 58%; height: auto;" />
162<p>Once the kafka-ip ingestion pipeline is successfully created, records from Kafka in <strong>kafkatopic</strong> are ingested into Mach5 index <strong>kafkaindex</strong>. As and when new data comes to the Kafka topic, it will get added into the Mach5 index</p>
163<h3 id="verify-data-ingestion" class="scroll-target">Verify data ingestion</h3>
164<p>Using Mach5 Dashboards - Dev Tools verify that the data from Kafka is ingested into Mach5</p>
165<ul>
166<li>Execute a <strong>count</strong> query on the Mach5 index to verify the number of ingested records. As expected, the count is 10 records. This is what was ingested into kafkatopic, refer to <a href="#prerequisites">Prerequisites</a></li>
167</ul>
168<img src="/images/docs/ingestingkafkadata/kafka-devtools-count.png" alt="DevToolsCount" style="max-width: 100%; height: auto;" />
169<ul>
170<li>Execute a <strong>search</strong>
170 query on the Mach5 index to verify the ingested records.</li>
171</ul>
172<img src="/images/docs/ingestingkafkadata/kafka-devtools-search.png" alt="DevToolsSearch" style="max-width: 100%; height: auto;" />
173<p>As expected, the output shows 10 ipdata records. This is what was ingested into kafkatopic, refer to <a href="#prerequisites">Prerequisites</a></p>
174<h3 id="disable-ingest-pipeline" class="scroll-target">Disable ingest pipeline</h3>
175<p>When not in use, ingest pipeline can be disabled. So any updates to source data are not reflected in Mach5</p>
176<img src="/images/docs/ingestingkafkadata/kafka-ip-list.png" alt="IPList" style="max-width: 78%; height: auto;" />
177<ul>
178<li>To Disable an existing pipeline, click on the Edit icon next to the the ingest pipeline, eg. <strong>kafka-ip</strong></li>
179<li>In the Edit ingest pipeline page, at end of all options, before <strong>Save</strong> button, deselect the <strong>Enabled</strong> checkbox</li>
180<li><strong>Save</strong> the ingest pipeline to take effect</li>
181<li>The <strong>kafka-ip</strong> ingest pipeline is now disabled. It will not read data from source to ingest data into Mach5. It can be enabled any time it needs to be re-used</li>
182</ul>
183<h2 id="kafka-tombstone-support-in-mach5" class="scroll-target">Kafka Tombstone Support in Mach5</h2><h3 id="tombstone-messages" class="scroll-target">Tombstone Messages</h3>
184<p>
184In Kafka, tombstone messages are messages with a valid, non-empty key and an empty/null payload (no value). Tombstone messages can be added to a topic by any producer and are often used in conjunction with compacted topics in order to delete previous instances of a particular key.</p>
185<p>Transform functions are optional in Mach5 ingest pipelines. However, when ingesting Kafka tombstone messages, a transform function is required to explicitly define the desired tombstone handling behavior. When Mach5 detects a tombstone message, the <code>meta.kafka.op</code> field is set to <code>"delete"</code> for the record in the ingest pipeline. This field can be interpreted by the transform function in order to produce one of two outcomes:</p>
186<ol>
187<li><strong>Delete</strong>: When the transform function returns a record containing a single <code>_id</code> field, the corresponding document in the index with the matching <code>_id</code> field (if it exists) will be deleted. Including any fields other than <code>_id</code> will result in an error.</li>
188<li><strong>Ignore</strong>: When the transform function returns the empty array <code>[]</code>, no action will be performed on the index.</li>
189</ol>
190<h3 id="examples" class="scroll-target">Examples</h3><h4 id="deleting-with-tombstone-messages" class="scroll-target">Deleting with Tombstone Messages</h4>
191<p>The following transform function shows how to (explicitly) interpret and act upon a delete operation.</p>
192<p>Note: The following example assumes that the Kafka record key is non-null and UTF-8–encoded, and that it maps directly to the document <code>_id</code>. This is a common pattern, but not a requirement. Applications with different key encodings or error-handling needs should decode and validate the key accordingly (for example, using try/catch).</p>
193<pre><code class="language-js">function transform(payload, meta) {
194  const kafka = meta?.kafka;
195  
196  // Assumes the key is not null and is a valid UTF8 string
197  const id = decodeUtf8(new Uint8Array(kafka.key));
198  
199  if (kafka?.op === "delete") {
200    return { _id: id };
201  } else {
202    // Further processing and transformation of payload
203    return { ...payload, _id: id};
204  }
205}
206
207const rec = transform(arg, meta);
208rec
209</code></pre>
210<p>A more compact version of this function can be written to take advantage of JavaScript object spreading semantics:</p>
211<pre><code class="language-js">function transform(payload, meta) {
212  const kafka = meta?.kafka;
213  const id = decodeUtf8(new Uint8Array(kafka.key));
214  return { ...payload, _id: id };
215}
216
217const rec = transform(arg, meta);
218rec
219</code></pre>
220<p>When the operation is a delete, <code>arg/payload</code> is <code>null</code>, so spreading is a no-op in JavaScript, producing a single object with the <code>_id</code> field as required.</p>
221<h4 id="ignoring-tombstone-messages" class="scroll-target">Ignoring Tombstone Messages</h4>
222<p>The following example shows how to explicitly ignore tombstone messages while continuing to process non-tombstone records:</p>
223<pre><code class="language-js">function transform(payload, meta) {
224  const kafka = meta?.kafka;
225
226  if (kafka?.op === "delete") {
227    return [];
228  }
229
230  // Normal processing for non-tombstone records
231  return payload;
232}
233</code></pre>
234<h2 id="troubleshooting" class="scroll-target">Troubleshooting</h2><h3 id="issue-ingest-pipeline-does-not-work-and-no-new-pods-are-being-spawned" class="scroll-target">Issue: Ingest pipeline does not work and no new pods are being spawned</h3>
235<ul>
236<li><strong>Symptom</strong>: The ingest pipeline appears to be configured but no data is ingested.</li>
237<li><strong>Cause</strong>: The Mach5 index referenced in the ingest pipeline does not exist.</li>
238<li><strong>Resolution</strong>: Ensure that the required Mach5 index is created before configuring or enabling the ingest pipeline.</li>
239</ul>
240</article></div></div></div><div class="w-[22rem] flex-shrink-0 overflow-y-auto scrollbar-thin scrollbar-thumb-transparent hover:scrollbar-thumb-gray-500 scrollbar-track-transparent hidden lg:block bg-[#FFFDF9] border-l border-[#31556F]/15"><div class="w-full flex-shrink-0 hidden lg:block mt-20"><div class="sticky top-[112px] px-4"><h3 class="text-sm font-bold uppercase tracking-wide mb-4 text-orange-600 pl-2">On this page</h3><!><link rel="stylesheet" href="/styles/github-markdown.min.css"><link rel="stylesheet" href="/styles/github.min.css"><link rel="stylesheet" href="/styles/override-github-markdown.min.css"><div class="markdown-body w-full px-4 py-6"><nav class="toc text-[#18222B] mb-8 pb-4 border-b border-[#31556F]/15"><ul><li style="margin-left: 1em;;"><a href="#prerequisites">Prerequisites</a></li><li style="margin-left: 1em;;"><a href="#connections">Connections</a></li><li style="margin-left: 2em;;"><a href="#add-a-new-connection">Add a new connection</a></li><li style="margin-left: 2em;;"><a href="#verify-new-connection">Verify new connection</a></li><li style="margin-left: 1em;;"><a href="#ingest-pipelines">Ingest Pipelines</a></li><li style="margin-left: 2em;;"><a href="#add-an-ingest-pipeline">Add an ingest pipeline</a></li><li style="margin-left: 2em;;"><a href="#verify-an-ingest-pipeline">Verify an ingest pipeline</a></li><li style="margin-left: 2em;;"><a href="#verify-data-ingestion">Verify data ingestion</a></li><li style="margin-left: 2em;;"><a href="#disable-ingest-pipeline">Disable ingest pipeline</a></li><li style="margin-left: 1em;;"><a href="#kafka-tombstone-support-in-mach5">Kafka Tombstone Support in Mach5</a></li><li style="margin-left: 2em;;"><a href="#tombstone-messages">Tombstone Messages</a></li><li style="margin-left: 2em;;"><a href="#examples">Examples</a></li><li style="margin-left: 3em;;"><a href="#deleting-with-tombstone-messages">Deleting with Tombstone Messages</a></li><li style="margin-left: 3em;;"><a href="#ignoring-tombstone-messages">Ignoring Tombstone Messages</a></li><li style="margin-left: 1em;;"><a href="#troubleshooting">Troubleshooting</a></li><li style="margin-left: 2em;;"><a href="#issue-ingest-pipeline-does-not-work-and-no-new-pods-are-being-spawned">Issue: Ingest pipeline does not work and no new pods are being spawned</a></li><!></ul></nav><!></div><div class="rounded-lg shadow-sm overflow-hidden border border-[#31556F]/15 bg-[#FFFDF9]"><div class="bg-[#FF5C1B] px-4 py-5 text-white"><h3 class="text-lg font-medium">Need Help?</h3><p class="mt-1 text-sm">Our team of experts is ready to assist you with your integration.</p></div><div class="px-4 py-5 bg-[#F7F3ED]"><div class="mb-6"><h4 class="text-sm font-medium text-[#18222B] mb-1">Training Sessions</h4><p class="text-sm text-[#52606C]">Get your team up to speed with personalized training.</p></div><a href="/schedule-a-demo" class="block bg-[#FF5C1B] hover:bg-[#F76D34] text-white text-center py-2 px-4 roun
240ded-md transition-colors">Contact Sales</a></div></div></div></div></div></main></div></main><footer class="bg-[#18222B] text-white px-4 sm:px-6 lg:px-8 py-12"><div class="max-w-7xl mx-auto"><div class="grid gap-10 md:grid-cols-[1.3fr_repeat(4,1fr)]"><div><a href="/" class="flex items-center gap-3"><img src="/images/mach5.svg" alt="Mach5" class="h-9 w-9 rounded-md"><span class="font-space-grotesk text-2xl font-semibold">Mach5</span></a><p class="mt-4 text-sm leading-relaxed text-[#C8D1D8] max-w-xs">Security data infrastructure for cybersecurity products.</p><a href="/schedule-a-demo" class="mt-5 inline-flex rounded-lg bg-[#FF5C1B] px-4 py-2 text-sm font-semibold text-white hover:bg-[#F76D34] transition-colors">Book a Demo</a></div><div><h3 class="text-sm font-semibold text-[#FF8E62] mb-3">Platform</h3><ul class="space-y-2"><li><a href="/platform" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Overview</a></li><li><a href="/platform/search-analytics" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Search &amp; Analytics</a></li><li><a href="/platform/streaming" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Streaming</a></li><li><a href="/platform/integrations" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Integrations</a></li><!></ul></div><div><h3 class="text-sm font-semibold text-[#FF8E62] mb-3">Workloads</h3><ul class="space-y-2"><li><a href="/solutions" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Workloads Hub</a></li><li><a href="/solutions/real-time-search" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Customer Search</a></li><li><a href="/solutions/log-analysis" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Log Analytics</a></li><li><a href="/resources/migrate-from-elasticsearch" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Elasticsearch Relief</a></li><li><a href="/docs/6.2.0/ingestion/kafka" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Streaming Pipelines</a></li><!></ul></div><div><h3 class="text-sm font-semibold text-[#FF8E62] mb-3">Customers</h3><ul class="space-y-2"><li><a href="/case-studies" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Customer Stories</a></li><li><a href="/case-studies/permiso" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Permiso</a></li><li><a href="/incidentbench" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Benchmarks</a></li><!></ul></div><div><h3 class="text-sm font-semibold text-[#FF8E62] mb-3">Resources</h3><ul class="space-y-2"><li><a href="/resources" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Resource Library</a></li><li><a href="/docs/6.2.0" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Docs</a></li><li><a href="/company" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Company</a></li><li><a href="/company/contact" class="text-sm text-[#C8D1D8] hover:text-white transition-colors">Contact</a></li><!></ul></div></div><div class="mt-10 pt-6 border-t border-white/10 flex flex-col md:flex-row gap-4 md:items-center md:justify-between text-sm text-[#C8D1D8]"><div>© 2021-2026 Mach5 Software, Inc. All rights reserved.</div><div class="flex flex-wrap gap-4"><a href="/privacy" class="hover:text-white">Privacy Notice</a><a href="/terms" class="hover:text-white">Terms of Service</a><button type="button" data-privacy-preferences-button class="hover:text-white">Cookie Settings</button></div></div></div></footer><section id="privacy-preferences-banner" aria-live="polite" aria-label="Privacy preferences" class="fixed inset-x-0 bottom-0 z-50 px-3 pb-3 sm:px-6"><div class="mx-auto max-w-4xl rounded-xl border border-[#31556F]/15 bg-[#FFFDF9]/95 shadow-2xl backdrop-blur"><div class="p-4 sm:p-5"><div class="flex flex-col gap-4 lg:flex-row lg:items-center lg:justify-between"><div class="max-w-2xl"><p class="text-[11px] font-semibold uppercase tracking-[0.18em] text-[#FF5C1B]">Analytics Cookies</p><h2 class="mt-1 text-lg font-semibold text-[#18222B]">Help us understand website usage.</h2><p class="mt-2 text-xs sm:text-sm leading-relaxed text-[#52606C]">Necessary storage remembers your choice. With your consent, Mach5 also uses PostHog analytics to measure website traffic and interactions.</p><p class="mt-1 text-xs leading-relaxed text-[#52606C]">Change this anytime from Cookie Settings in the footer. <a href="/privacy" class="text-[#FF5C1B] underline hover:text-[#D94C14]">Privacy Notice</a>.</p></div><div class="flex flex-col gap-2 sm:flex-row lg:min-w-[300px] lg:justify-end"><button type="button" data-cookie-action="reject" class="inline-flex items-center justify-center rounded-lg border border-[#31556F]/25 px-4 py-2.5 text-sm font-semibold text-[#31556F] transition-colors hover:border-[#31556F]/45 hover:bg-[#F7F3ED]">Reject Analytics</button><button type="button" data-cookie-action="accept" class="inline-flex items-center justify-center rounded-lg bg-[#FF5C1B] px-4 py-2.5 text-sm font-semibold text-white transition-colors hover:bg-[#F76D34]">Accept Analytics</button></div></div></div></div></section>
240<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"e975e51e1e10496f87a21c9eba4c9d40","r":1,"spa":2}' crossorigin="anonymous"></script>
240
241</body></html>
241<script nonce="9ak8jQ8_yXW6RprR1oxjTw">__RESOLVED_RESOURCES=[];__SERIALIZED_ERRORS=[];__PENDING_RESOURCES=[];__RESOURCE_RESOLVERS=[];</script>
241<script nonce="9ak8jQ8_yXW6RprR1oxjTw">__INCOMPLETE_CHUNKS=[];</script>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.