1"use strict";(self.webpackChunkcilium_io=self.webpackChunkcilium_io||[]).push([[306],{3900:function(n,e,s){var a=s(8453),t=s(6540),l=s(6452);function o(n){const e=Object.assign({h1:"h1",p:"p",strong:"strong",em:"em",ul:"ul",li:"li",a:"a",span:"span",h2:"h2",h3:"h3",ol:"ol",h4:"h4",table:"table",thead:"thead",tr:"tr",th:"th",tbody:"tbody",td:"td"},(0,a.RP)(),n.components),{BlogAuthor:s}=e;return s||function(n,e){throw new Error("Expected "+(e?"component":"object")+" `"+n+"` to be defined: you likely forgot to import, pass, or provide it.")}("BlogAuthor",!0),t.createElement(t.Fragment,null,t.createElement(e.h1,null,"Troubleshooting Kubernetes Networking"),"\n",t.createElement(e.h1,null,"1.0 Introduction"),"\n",t.createElement(e.p,null,"Networking is notoriously one of the most common failure points encountered in workloads deployed within a Kubernetes environment. Because a cluster consists of hundreds or thousands of ephemeral compute resources (pods, nodes, and services) constantly spinning up, shutting down, and scaling, the underlying network layer bears the immense responsibility of keeping everything connected. When an application breaks in Kubernetes, ",t.createElement(e.strong,null,t.createElement(e.em,null,"it is probably a network issue")),"."),"\n",t.createElement(e.p,null,"Before container orchestration, it was somewhat easy to troubleshoot outages led by networking challenges. Servers had static IP addresses, physical switches had predictable ports, and firewalls relied on static rules. Kubernetes completely shatters this model.\nThe challenges of debugging Kubernetes networks stem directly from its highly dynamic architecture:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,t.createElement(e.strong,null,"Ephemeral IP Addresses"),": Pod IPs change every time a deployment is scaled, restarted, or rescheduled to a new node. Tracking down a connection failure using a raw IP address from an hour-old log file is often useless, as that IP may now belong to a completely different application."),"\n",t.createElement(e.li,null,t.createElement(e.strong,null,"Complex Abstractions"),": Traffic routing relies on overlapping, virtualized abstractions like ClusterIPs, NodePorts, Ingress controllers, and CoreDNS."),"\n",t.createElement(e.li,null,t.createElement(e.strong,null,"The iptables Black Box"),": Historically, Kubernetes relied heavily on kube-proxy and massive, sequential lists of iptables rules to route traffic. When a packet is unexpectedly dropped, tracing it through thousands of generic rules to find the exact misconfiguration is incredibly difficult and completely lacks appliscreen readercation-level context."),"\n"),"\n",t.createElement(e.p,null,"Misconfiguration is consistently reported as the leading source of Kubernetes incidents ",t.createElement(e.a,{href:"https://www.redhat.com/en/resources/kubernetes-adoption-security-market-trends-overview"},"[Red Hat 2024]"),", while networking and DNS faults, though less frequent, are disproportionately difficult to localise."),"\n",t.createElement(e.h1,null,"2.0 OSI Stack"),"\n",t.createElement(e.p,null,"Kubernetes abstracts away physical infrastructure, but the OSI model does not disappear; it shifts. Every packet sent between pods, services, nodes, or external clients still traverses all seven layers. What changes is who owns each layer: the Linux kernel, the CNI plugin, kube-proxy (or its replacement), CoreDNS, Ingress controllers, and application code. Each operates at different OSI levels, and when something breaks, the layer boundary is usually where you start your investigation."),"\n",t.createElement(e.p,null,"Cilium's datapath is unique because it operates simultaneously at Layers 2-7 through kernel hooks, including TC (traffic control), XDP (Layer 2/3), kprobes, and socket-level filters (Layer 4), as well as Envoy-integrated Layer 7 parsing. This means a single Cilium-equipped cluster can surface and debug issues at every OSI layer through one unified toolchain: Hubble, cilium-dbg, and the BPF map inspection commands."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/bc10e467bf39f6c3cbc549034218c94e/44899/osi-stack.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 50%; position: relative; bottom: 0; left: 0; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/bc10e467bf39f6c3cbc549034218c94e/2ff5b/osi-stack.webp 252w,\n/static/bc10e467bf39f6c3cbc549034218c94e/4d583/osi-stack.webp 504w,\n/static/bc10e467bf39f6c3cbc549034218c94e/905a7/osi-stack.webp 1008w,\n/static/bc10e467bf39f6c3cbc549034218c94e/bb9f8/osi-stack.webp 1512w,\n/static/bc10e467bf39f6c3cbc549034218c94e/5aae5/osi-stack.webp 1999w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/bc10e467bf39f6c3cbc549034218c94e/019e0/osi-stack.png 252w,\n/static/bc10e467bf39f6c3cbc549034218c94e/0dcb2/osi-stack.png 504w,\n/static/bc10e467bf39f6c3cbc549034218c94e/832a9/osi-stack.png 1008w,\n/static/bc10e467bf39f6c3cbc549034218c94e/19357/osi-stack.png 1512w,\n/static/bc10e467bf39f6c3cbc549034218c94e/44899/osi-stack.png 1999w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/bc10e467bf39f6c3cbc549034218c94e/832a9/osi-stack.png"\n alt="The OSI Stack"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",t.createElement(e.p,null,t.createElement(e.em,null,'Figure 1: OSI stack. Image credit: Cloudflare, "What is the OSI Model?" Cloudflare Learning Center. ',t.createElement(e.a,{href:"https://www.cloudflare.com/learning/ddos/glossary/open-systems-interconnection-model-osi/"},"https://www.cloudflare.com/learning/ddos/glossary/open-systems-interconnection-model-osi/"))),"\n",t.createElement(e.h2,null,"Physical Layer: Node and NIC Failures"),"\n",t.createElement(e.p,null,"Although Kubernetes doesn't manage physical hardware directly, L1 failures surface immediately in cluster networking."),"\n",t.createElement(e.h3,null,"Symptoms"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"Node becomes NotReady in ",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">kubectl get nodes</code>'}})),"\n",t.createElement(e.li,null,"All pods on the affected node become unreachable"),"\n",t.createElement(e.li,null,"cilium status reports Cluster health: 0/N reachable for the affected node"),"\n"),"\n",t.createElement(e.p,null,"Common causes in cloud and on-prem"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"MTU mismatch between the host NIC and the overlay tunnel. Cilium's VXLAN encapsulation adds 50 bytes of overhead. If the underlay MTU is 1500 and Cilium uses 1500 for pod traffic, frames are silently dropped during fragmentation."),"\n",t.createElement(e.li,null,"NIC queue saturation on high-throughput nodes."),"\n"),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/network/concepts/routing/"},"Read More about MTU Overhead")),"\n",t.createElement(e.p,null,"Cilium Commands"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Inspect the route MTU on the cilium_host interface</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- ip link show cilium_host</span>\n<span class="token number">4</span>
1: cilium_host@cilium_net: <span class="token operator"><</span>BROADCAST,MULTICAST,NOARP,UP,LOWER_UP<span class="token operator">></span> mtu <span class="token number">1500</span> qdisc noqueue state UP mode DEFAULT group default qlen <span class="token number">1000</span>\n link/ether 5e:9d:ba:cf:a2:84 brd ff:ff:ff:ff:ff:ff\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check MTU configuration used by Cilium</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec -it ds/cilium -- cilium status --verbose | grep -i mtu</span>\n â âââ mtu\n â â âââ job-endpoint-mtu-updater <span class="token punctuation">[</span>OK<span class="token punctuation">]</span> Endpoint MTU updated <span class="token punctuation">(</span>14m, x1<span class="token punctuation">)</span>\n â â âââ job-mtu-updater <span class="token punctuation">[</span>OK<span class="token punctuation">]</span> MTU updated <span class="token punctuation">(</span><span class="token number">1500</span><span class="token punctuation">)</span> <span class="token punctuation">(</span>14m, x1<span class="token punctuation">)</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Verify MTU on the overlay interface (if using tunnel mode)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- ip link show cilium_vxlan</span>\n<span class="token number">5</span>: cilium_vxlan: <span class="token operator"><</span>BROADCAST,MULTICAST,UP,LOWER_UP<span class="token operator">></span> mtu <span class="token number">1500</span> qdisc noqueue state UNKNOWN mode DEFAULT group default\n link/ether 3a:0d:29:2d:1f:14 brd ff:ff:ff:ff:ff:ff\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Data Link: ARP, MAC Resolution, Bridge Issues"),"\n",t.createElement(e.p,null,"With symptoms like pods on the same node failing to communicate, and intermittent packet loss between pods on adjacent nodes."),"\n",t.createElement(e.p,null,"Common causes"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"ARP flux: nodes with multiple interfaces can reply to ARP requests from the wrong interface, causing asymmetric routes."),"\n",t.createElement(e.li,null,"Bridge netfilter disabled: if net.bridge.bridge-nf-call-iptables is off, kube-proxy-based clusters break at L2/L3 boundary (less relevant with Cilium's full eBPF replacement, but still matters during migration)"),"\n",t.createElement(e.li,null,"Cilium is using veth device mode, where the veth peer goes missing after pod restart, due to race conditions."),"\n"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># List all endpoints and their MAC / IP state</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg endpoint list</span>\nENDPOINT POLICY <span class="token punctuation">(</span>ingress<span class="token punctuation">)</span> POLICY <span class="token punctuation">(</span>egress<span class="token punctuation">)</span> IDENTITY LABELS <span class="token punctuation">(</span>source:key<span class="token punctuation">[</span><span class="token operator">=</span>value<span class="token punctuation">]</span><span class="token punctuation">)</span> IPv6 IPv4 STATUS\n ENFORCEMENT ENFORCEMENT\n<span class="token number">2179</span>
1 Disabled Disabled <span class="token number">1</span> reserved:host ready\n<span class="token number">2216</span> Disabled Disabled <span class="token number">8</span> reserved:ingress <span class="token number">10.244</span>.2.108 ready\n<span class="token number">2450</span> Disabled Disabled <span class="token number">4</span> reserved:health <span class="token number">10.244</span>.2.19 ready\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check if the veth pair for a specific endpoint is intact</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- ip link show | grep lxc</span>\n<span class="token number">7</span>: lxc_health@if6: <span class="token operator"><</span>BROADCAST,MULTICAST,UP,LOWER_UP<span class="token operator">></span> mtu <span class="token number">1500</span> qdisc noqueue state UP mode DEFAULT group default qlen <span class="token number">1000</span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Network Layer: IP Routing, IPAM, Overlay Tunnels"),"\n",t.createElement(e.p,null,'This is the most frequently hit layer in Kubernetes networking incidents. Most "pod can\'t reach pod on another node" issues live here.'),"\n",t.createElement(e.p,null,"Most of the time, issues in this layer are characterised by a timeout in ping between two pods, an internode traffic drop while intra-node traffic is ok, and CoreDNS unreachable with a timeout."),"\n",t.createElement(e.p,null,"Common causes:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"IPAM exhaustion: Cilium's default cluster-pool IPAM allocates pod CIDRs per node. If the CIDR is exhausted, new pods fail to launch with IP assignment errors."),"\n",t.createElement(e.li,null,"Missing or stale BPF route entries after node restarts."),"\n",t.createElement(e.li,null,"VXLAN tunnel mismatch: Two nodes on different overlay VTEP configurations can't establish a tunnel."),"\n",t.createElement(e.li,null,"IP masquerade misconfigured: pods can reach the cluster network but not external services because SNAT rules aren't applied correctly."),"\n"),"\n",t.createElement(e.p,null,"Cilium Commands\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check IPAM allocation state</code>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipmasq list</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipmasq list</span>\nNo entries found.\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Inspect the IP address manager</code>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">kubectl -n kube-system exec ds/cilium -- cilium-dbg ip list</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg ip list</span>\nIP IDENTITY SOURCE\n<span class="token number">0.0</span>.0.0/0 reserved:world\n<span class="token number">10.244</span>.0.0/24 reserved:world\n<span class="token number">10.244</span>.1.0/24 reserved:world\n<span class="token number">10.244</span>.0.76/32 reserved:ingress\n<span class="token number">10.244</span>.0.127/32 reserved:remote-node\n<span class="token number">10.244</span>.0.139/32 reserved:health\n<span class="token number">10.244</span>.1.112/32 reserved:remote-node\n<span class="token number">10.244</span>.1.230/32 reserved:health\n<span class="token number">10.244</span>.1.251/32 reserved:ingress\n<span class="token number">10.244</span>.2.17/32 k8s:app<span class="token operator">=</span>local-path-provisioner custom-resource\n k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>
1local-path-storage\n k8s:io.cilium.k8s.policy.cluster<span class="token operator">=</span>kind-kind\n k8s:io.cilium.k8s.policy.serviceaccount<span class="token operator">=</span>local-path-provisioner-service-account\n k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>local-path-storage\n<span class="token number">10.244</span>.2.72/32 reserved:host\n<span class="token number">10.244</span>.2.73/32 reserved:ingress\n<span class="token number">10.244</span>.2.76/32 k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>kube-system custom-resource\n k8s:io.cilium.k8s.policy.cluster<span class="token operator">=</span>kind-kind\n k8s:io.cilium.k8s.policy.serviceaccount<span class="token operator">=</span>coredns\n k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>kube-system\n k8s:k8s-app<span class="token operator">=</span>kube-dns\n<span class="token number">10.244</span>.2.127/32 reserved:health\n<span class="token number">10.244</span>.2.149/32 k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>kube-system custom-resource\n k8s:io.cilium.k8s.policy.cluster<span class="token operator">=</span>kind-kind\n k8s:io.cilium.k8s.policy.serviceaccount<span class="token operator">=</span>coredns\n k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>kube-system\n k8s:k8s-app<span class="token operator">=</span>kube-dns\n<span class="token number">172.18</span>.0.2/32 reserved:remote-node\n<span class="token number">172.18</span>.0.3/32 reserved:kube-apiserver\n reserved:remote-node\n<span class="token number">172.18</span>.0.4/32 reserved:host\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># List all BPF routing entries</code>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipcache list</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipcache list</span>\nIP PREFIX/ADDRESS IDENTITY\n<span class="token number">10.244</span>.0.139/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">4</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.3 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.1.0/24 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">2</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.2 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.2.72/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">1</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">10.244</span>
1.0.0/24 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">2</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.3 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.1.112/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">6</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.2 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.2.17/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">3388</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">172.18</span>.0.2/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">6</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">172.18</span>.0.3/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">7</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">172.18</span>.0.4/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">1</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">10.244</span>.1.230/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">4</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.2 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>
1.2.73/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">8</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">10.244</span>.2.149/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">48339</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">10.244</span>.0.76/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">8</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.3 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.0.127/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">6</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.3 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.1.251/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">8</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.2 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\n<span class="token number">10.244</span>.2.76/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">48339</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">10.244</span>.2.127/32 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">4</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\n<span class="token number">0.0</span>
1.0.0/0 <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">2</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">0.0</span>.0.0 <span class="token assign-left variable">flags</span><span class="token operator">=</span><span class="token operator"><</span>none<span class="token operator">></span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check the ipcache â maps pod IPs to identities and tunnel endpoints</code>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text">kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipcache get <pod-ip></code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ipcache get 10.244.1.45</span>\n<span class="token number">10.244</span>.1.45 maps to identity <span class="token assign-left variable">identity</span><span class="token operator">=</span><span class="token number">6026</span> <span class="token assign-left variable">encryptkey</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">tunnelendpoint</span><span class="token operator">=</span><span class="token number">172.18</span>.0.2 <span class="token assign-left variable">flags</span><span class="token operator">=</span>hastunnel\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Verify tunnel map (VXLAN/Geneve mode)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -c cilium-agent -- cilium-dbg node list</span>\nName IPv4 Address Endpoint CIDR IPv6 Address Endpoint CIDR Source\nkind-kind/kind-control-plane <span class="token number">172.18</span>.0.3 <span class="token number">10.244</span>.0.0/24 custom-resource\nkind-kind/kind-worker <span class="token number">172.18</span>.0.4 <span class="token number">10.244</span>.2.0/24 <span class="token builtin class-name">local</span>\nkind-kind/kind-worker2 <span class="token number">172.18</span>.0.2 <span class="token number">10.244</span>.1.0/24 custom-resource\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Run Cilium\'s built-in connectivity test</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># cilium connectivity test</span>\nâ¹ï¸ Monitor aggregation detected, will skip some flow validation steps\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Creating namespace cilium-test-1 <span class="token keyword">for</span> connectivity check<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying echo-same-node service<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying DNS <span class="token builtin class-name">test</span>
1 server configmap<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying same-node deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client2 deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client3 deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying echo-other-node service<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying other-node deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>host-netns<span class="token punctuation">]</span> Deploying kind-kind daemonset<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>host-netns-non-cilium<span class="token punctuation">]</span> Deploying kind-kind daemonset<span class="token punctuation">..</span>.\nâ¹ï¸ Skipping tests that require a <span class="token function">node</span> Without Cilium\nâ <span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Waiting <span class="token keyword">for</span> deployment cilium-test-1/client to become ready<span class="token punctuation">..</span>.\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Hubble (L3 drops)"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch all dropped flows at L3 with reason</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --type drop --last 200 -o jsonpb | jq \'.flow | {src: .source.pod_name, dst: .destination.pod_name, reason: .drop_reason_desc}\'</span>\n<span class="token punctuation">{</span>\n <span class="token string">"src"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"dst"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"reason"</span><span class="token builtin class-name">:</span> <span class="token string">"UNSUPPORTED_L3_PROTOCOL"</span>\n<span class="token punctuation">}</span>\n<span class="token punctuation">{</span>\n <span class="token string">"src"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"dst"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"reason"</span><span class="token builtin class-name">:</span> <span class="token string">"UNSUPPORTED_L3_PROTOCOL"</span>\n<span class="token punctuation">}</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch traffic between two specific pods</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --from-pod default/frontend --to-pod default/backend --follow</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.h2),"\n",t.createElement(e.h2,null,"Transport Layer: TCP/UDP Services"),"\n",t.createElement(e.p,null,"Network issues in this layer are characterised by failure in TCP connections despite being able to ping it, NodePort services being reachable from only a few nodes but not others, uneven distribution of traffic in Load Balancer services, and exhausted connections in the nf_conntrack table.\nCommon Causes:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"Cilium service map out of sync with Kubernetes Endpoints: endpoints change faster than Cilium's reconciliation loop, especially during rolling deploys."),"\n",t.createElement(e.li,null,"kube-proxy running alongside Cilium in partial replacement mode, creating conflicting NAT rules."),"\n",t.createElement(e.li,null,"BPF NAT map full (default 524288 entr
1ies) on very high-connection-rate nodes"),"\n",t.createElement(e.li,null,"Session affinity misconfiguration is causing stickiness to terminate pods."),"\n"),"\n",t.createElement(e.p,null,"Cilium Commands"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># List all Cilium-managed services and their backends</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg service list</span>\nID Frontend Service Type Backend\n<span class="token number">1</span> <span class="token number">10.96</span>.0.1:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">2</span> <span class="token number">0.0</span>.0.0:30464/TCP NodePort\n<span class="token number">4</span> <span class="token number">0.0</span>.0.0:30977/TCP NodePort\n<span class="token number">6</span> <span class="token number">10.96</span>.183.162:80/TCP ClusterIP\n<span class="token number">7</span> <span class="token number">10.96</span>.183.162:443/TCP ClusterIP\n<span class="token number">8</span> <span class="token number">10.96</span>.248.144:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.3:4244/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">9</span> <span class="token number">0.0</span>.0.0:31234/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">11</span> <span class="token number">10.96</span>.218.16:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">12</span> <span class="token number">0.0</span>.0.0:31235/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">14</span> <span class="token number">10.96</span>.108.211:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">15</span> <span class="token number">10.96</span>.0.10:53/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">16</span> <span class="token number">10.96</span>
1.0.10:53/UDP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:53/UDP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:53/UDP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">17</span> <span class="token number">10.96</span>.0.10:9153/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:9153/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:9153/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">18</span> <span class="token number">172.18</span>.255.200:80/TCP LoadBalancer\n<span class="token number">19</span> <span class="token number">172.18</span>.255.200:443/TCP LoadBalancer\n<span class="token number">20</span> <span class="token number">10.96</span>.141.89:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.2.163:80/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check a specific service\'s backend health</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg service list</span>\nID Frontend Service Type Backend\n<span class="token number">1</span> <span class="token number">10.96</span>.0.1:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">2</span> <span class="token number">0.0</span>.0.0:30464/TCP NodePort\n<span class="token number">4</span> <span class="token number">0.0</span>.0.0:30977/TCP NodePort\n<span class="token number">6</span> <span class="token number">10.96</span>.183.162:80/TCP ClusterIP\n<span class="token number">7</span> <span class="token number">10.96</span>.183.162:443/TCP ClusterIP\n<span class="token number">8</span> <span class="token number">10.96</span>.248.144:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.3:4244/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">9</span> <span class="token number">0.0</span>.0.0:31234/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">11</span> <span class="token number">10.96</span>.218.16:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">12</span> <span class="token number">0.0</span>
1.0.0:31235/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">14</span> <span class="token number">10.96</span>.108.211:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">15</span> <span class="token number">10.96</span>.0.10:53/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">16</span> <span class="token number">10.96</span>.0.10:53/UDP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:53/UDP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:53/UDP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">17</span> <span class="token number">10.96</span>.0.10:9153/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:9153/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:9153/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">18</span> <span class="token number">172.18</span>.255.200:80/TCP LoadBalancer\n<span class="token number">19</span> <span class="token number">172.18</span>.255.200:443/TCP LoadBalancer\n<span class="token number">20</span> <span class="token number">10.96</span>.141.89:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.2.163:80/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Inspect the BPF load balancer map (frontend VIP â backends)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf lb list</span>\nSERVICE ADDRESS BACKEND ADDRESS <span class="token punctuation">(</span>REVNAT_ID<span class="token punctuation">)</span> <span class="token punctuation">(</span>SLOT<span class="token punctuation">)</span>\n<span class="token number">10.96</span>.218.16:80/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>
1 <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span><span class="token number">11</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token number">10.96</span>.248.144:443/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token number">172.18</span>.0.3:4244/TCP <span class="token punctuation">(</span><span class="token number">8</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token number">10.96</span>.141.89:80/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token number">10.244</span>.2.163:80/TCP <span class="token punctuation">(</span><span class="token number">20</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token number">10.96</span>.0.10:53/TCP <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">15</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\n<span class="token number">10.96</span>.108.211:0/ANY <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\n<span class="token number">0.0</span>.0.0:31235/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span><span class="token number">12</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token number">10.96</span>.141.89:0/ANY <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\n<span class="token number">172.18</span>.0.3:31234/TCP <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">10</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>NodePort<span class="token punctuation">]</span>\n<span class="token number">172.18</span>.0.3:31235/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span><span class="token number">13</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token punctuation">..</span><span class="token punctuation">..</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check BPF NAT table (active SNAT mappings)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf nat list</span>\nTCP OUT <span class="token number">172.18</span>.0.3:34178 -<span class="token operator">></span> <span class="token number">54.234</span>.221.194:443 XLATE_SRC <span class="token number">172.18</span>.0.3:34178 <span class="token assign-left variable">Created</span><span class="token operator">=</span>408sec ago <span class="token assign-left variable">NeedsCT</span><span class="token operator">=</span><span class="token number">1</span>\nTCP IN <span class="token number">54.234</span>.221.194:443 -<span class="token operator">></span> <span class="token number">172.18</span>.0.3:34178 XLATE_DST <span class="token number">172.18</span>.0.3:34178 <span class="token assign-left variable">Created</span><span class="token operator">=</span>408sec ago <span class="token assign-left variable">NeedsCT</span><span class="token operator">=</span><span class="token number">1</span>\nTCP IN <span class="token number">172.18</span>.0.4:6443 -<span class="token operator">></span> <span class="token number">172.18</span>.0.3:58624 XLATE_DST <span class="token number">172.18</span>.0.3:58624 <span class="token assign-left variable">Created</span><span class="token operator">=</span>667sec ago <span class="token assign-left variable">NeedsCT</span><span class="token operator">=</span><span class="token number">1</span>\nTCP OUT <span class="token number">172.18</span>.0.3:59112 -<span class="token operator">></span> <span class="token number">172.18</span>.0.4:4240 XLATE_SRC <span class="token number">172.18</span>.0.3:59112 <span class="token assign-left variable">Created</span><span class="token operator">=</span>297sec ago <span class="token assign-left variable">NeedsCT</span><span class="token operator">=</span><span class="token number">1</span>\nTCP OUT <span class="token number">172.18</span>.0.3:54010 -<span class="token operator">></span> <span class="token number">13.249</span>.228.49:443 XLATE_SRC <span class="token number">172.18</span>.0.3:54010 <span class="token assign-left variable">Created</span><span class="token operator">=</span>406sec ago <span class="token assign-left variable">NeedsCT</span><span class="token operator">=</span><span class="token number">1</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check connection tracking table</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ct list global</span>\nTCP OUT <span class="token number">172.18</span>.0.3:58624 -<span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443 <span class="token assign-left variable">expires</span><span class="token operator">=</span><span class="token number">19466</span> <span class="token assign-left variable">Packets</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">Bytes</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">RxFlagsSeen</span><span class="token operator">=</span>0x18 <span class="token assign-left variable">
1LastRxReport</span><span class="token operator">=</span><span class="token number">11461</span> <span class="token assign-left variable">TxFlagsSeen</span><span class="token operator">=</span>0x1a <span class="token assign-left variable">LastTxReport</span><span class="token operator">=</span><span class="token number">11461</span> <span class="token assign-left variable">Flags</span><span class="token operator">=</span>0x0010 <span class="token punctuation">[</span> SeenNonSyn <span class="token punctuation">]</span> <span class="token assign-left variable">RevNAT</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">SourceSecurityID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">BackendID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">NatPort</span><span class="token operator">=</span><span class="token number">0</span>\nTCP OUT <span class="token number">10.244</span>.2.148:38566 -<span class="token operator">></span> <span class="token number">10.244</span>.0.34:4240 <span class="token assign-left variable">expires</span><span class="token operator">=</span><span class="token number">19462</span> <span class="token assign-left variable">Packets</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">Bytes</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">RxFlagsSeen</span><span class="token operator">=</span>0x1a <span class="token assign-left variable">LastRxReport</span><span class="token operator">=</span><span class="token number">11462</span> <span class="token assign-left variable">TxFlagsSeen</span><span class="token operator">=</span>0x1a <span class="token assign-left variable">LastTxReport</span><span class="token operator">=</span><span class="token number">11462</span> <span class="token assign-left variable">Flags</span><span class="token operator">=</span>0x0010 <span class="token punctuation">[</span> SeenNonSyn <span class="token punctuation">]</span> <span class="token assign-left variable">RevNAT</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">SourceSecurityID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">BackendID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">NatPort</span><span class="token operator">=</span><span class="token number">0</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Compare Kubernetes services with what Cilium has programmed</code>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># (discrepancies = sync lag or bug)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get svc --all-namespaces</span>\nNAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT<span class="token punctuation">(</span>S<span class="token punctuation">)</span> AGE\ndefault backend-service ClusterIP <span class="token number">10.96</span>.141.89 <span class="token operator"><</span>none<span class="token operator">></span> <span class="token number">80</span>/TCP 10m\ndefault kubernetes ClusterIP <span class="token number">10.96</span>.0.1 <span class="token operator"><</span>none<span class="token operator">></span> <span class="token number">443</span>/TCP 3h11m\nkube-system cilium-envoy ClusterIP None <sp
1an class="token operator"><</span>none<span class="token operator">></span> <span class="token number">9964</span>/TCP 15m\nkube-system cilium-ingress LoadBalancer <span class="token number">10.96</span>.183.162 <span class="token number">172.18</span>.255.200 <span class="token number">80</span>:30977/TCP,443:30464/TCP 15m\nkube-system hubble-metrics ClusterIP None <span class="token operator"><</span>none<span class="token operator">></span> <span class="token number">9965</span>/TCP 15m\n<span class="token punctuation">..</span><span class="token punctuation">..</span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg service list</span>\nID Frontend Service Type Backend\n<span class="token number">1</span> <span class="token number">10.96</span>.0.1:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">2</span> <span class="token number">0.0</span>.0.0:30464/TCP NodePort\n<span class="token number">4</span> <span class="token number">0.0</span>.0.0:30977/TCP NodePort\n<span class="token number">6</span> <span class="token number">10.96</span>.183.162:80/TCP ClusterIP\n<span class="token number">7</span> <span class="token number">10.96</span>.183.162:443/TCP ClusterIP\n<span class="token number">8</span> <span class="token number">10.96</span>.248.144:443/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">172.18</span>.0.3:4244/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">9</span> <span class="token number">0.0</span>.0.0:31234/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">11</span> <span class="token number">10.96</span>.218.16:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.248:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">12</span> <span class="token number">0.0</span>.0.0:31235/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">14</span> <span class="token number">10.96</span>.108.211:80/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.174:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">15</span> <span class="token number">10.96</span>.0.10:53/TCP ClusterIP <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.82:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>
1\n <span class="token number">2</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.1.102:53/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token punctuation">..</span>.</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Hubble for L4 Drops"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch TCP-specific drops</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --type drop --protocol TCP --follow</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Filter by specific port</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --to-port 80 --follow</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check policy verdicts at L4</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --type policy-verdict --follow</span>\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Session Layer: Connection Tracking and Keep-Alive Issues"),"\n",t.createElement(e.p,null,"Kubernetes doesn't surface a dedicated session layer, but TCP session management is a real operational concern in long-lived microservice connections."),"\n",t.createElement(e.p,null,"Connections appear active in the application but not in BPF ConTrack Maps."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"Common Causes:")),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"BPF connection tracking idle timeout (default 300s for TCP established) terminates long-lived connections silently."),"\n",t.createElement(e.li,null,"NAT timeout is too aggressive for idle database connections."),"\n",t.createElement(e.li,null,"TCP keep-alive is not configured at the application level, and connections silently close mid-stream."),"\n"),"\n",t.createElement(e.p,null,"Cilium Commands"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># View CT timeouts and current CT table</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf ct list global | head -50</span>\nTCP IN <span class="token number">10.244</span>.1.46:56068 -<span class="token operator">></span> <span class="token number">10.244</span>.2.219:4240 <span class="token assign-left variable">expires</span><span class="token operator">=</span><span class="token number">11750</span> <span class="token assign-left variable">Packets</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">Bytes</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">RxFlagsSeen</span><span class="token operator">=</span>0x1b <span class="token assign-left variable">LastRxReport</span><span class="token operator">=</span><span class="token number">11740</span> <span class="token assign-left variable">TxFlagsSeen</span><span class="token operator">=</span>0x1b <span class="token assign-left variable">LastTxReport</span><span class="token operator">=</span><span class="token number">11740</span> <span class="token assign-left variable">Flags</span><span class="token operator">=</span>0x0413 <span class="token punctuation">[</span> RxClosing TxClosing SeenNonSyn FromTunnel <span class="token punctuation">]</span> <span class="token assign-left variable">RevNAT</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">SourceSecurityID</span><span class="token operator">=</span><span class="token number">6</span> <span class="token assign-left variable">BackendID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">NatPort</span><span class="token operator">=</span><span class="token number">0</span>\nTCP OUT <span class="token number">10.244</span>.2.148:44772 -<span class="token operator">></span> <span class="token number">10.244</span>.1.106:4240 <span class="token assign-left variable">expires</span><span class="token operator">=</span><span class="token number">11711</span> <span class="token assign-left variable">Packets</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">Bytes</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">RxFlagsSeen</span><span class="token operator">=</span>0x1b <span class="token assign-left variable">
1LastRxReport</span><span class="token operator">=</span><span class="token number">11701</span> <span class="token assign-left variable">TxFlagsSeen</span><span class="token operator">=</span>0x1b <span class="token assign-left variable">LastTxReport</span><span class="token operator">=</span><span class="token number">11701</span> <span class="token assign-left variable">Flags</span><span class="token operator">=</span>0x0013 <span class="token punctuation">[</span> RxClosing TxClosing SeenNonSyn <span class="token punctuation">]</span> <span class="token assign-left variable">RevNAT</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">SourceSecurityID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">BackendID</span><span class="token operator">=</span><span class="token number">0</span> <span class="token assign-left variable">NatPort</span><span class="token operator">=</span><span class="token number">0</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check Cilium config for current CT settings</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg config | grep -i conntrack</span>\nConntrackAccounting <span class="token builtin class-name">:</span> Disabled\nroot@server:~<span class="token comment">#</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Presentation Layer: TLS, mTLS, Certificate Issues"),"\n",t.createElement(e.p,null,"This is normally characterised by TLS handshake timeout or certificate verification failed; mTLS between pods fails after certificate rotation."),"\n",t.createElement(e.p,null,"Common Causes"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"Cilium WireGuard encryption is enabled on some nodes but not all (mixed encryption state)."),"\n",t.createElement(e.li,null,"certificate expiry in the SPIFFE/SPIRE integration causes identity verification failures."),"\n",t.createElement(e.li,null,"Ingress TLS secret missing or expired, causing 503 from the Ingress controller."),"\n"),"\n",t.createElement(e.p,null,"Commands"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check WireGuard encryption status on all nodes</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg encrypt status</span>\nEncryption: Disabled\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check node-to-node encryption state</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg node list</span>\nName IPv4 Address Endpoint CIDR IPv6 Address Endpoint CIDR Source\nkind-kind/kind-control-plane <span class="token number">172.18</span>.0.4 <span class="token number">10.244</span>.0.0/24 custom-resource\nkind-kind/kind-worker <span class="token number">172.18</span>.0.3 <span class="token number">10.244</span>.2.0/24 <span class="token builtin class-name">local</span>\nkind-kind/kind-worker2 <span class="token number">172.18</span>.0.2 <span class="token number">10.244</span>.1.0/24 custom-resource\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Hubble for TLS Drops"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell"><span class="token comment"># Watch flows with encryption status</span>\nhubble observe <span class="token parameter variable">--follow</span> <span class="token parameter variable">-o</span> jsonpb <span class="token operator">|</span> jq <span class="token string">\'select(.flow.is_reply == false) | {src: .flow.source.pod_name, dst: .flow.destination.pod_n
1ame, encrypted: .flow.traffic_direction}\'</span>\n\n\n<span class="token punctuation">{</span>\n <span class="token string">"src"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"dst"</span><span class="token builtin class-name">:</span> <span class="token string">"hubble-relay-5bbf66658c-k6958"</span>,\n <span class="token string">"encrypted"</span><span class="token builtin class-name">:</span> <span class="token string">"INGRESS"</span>\n<span class="token punctuation">}</span>\n<span class="token punctuation">{</span>\n <span class="token string">"src"</span><span class="token builtin class-name">:</span> null,\n <span class="token string">"dst"</span><span class="token builtin class-name">:</span> <span class="token string">"hubble-relay-5bbf66658c-k6958"</span>,\n <span class="token string">"encrypted"</span><span class="token builtin class-name">:</span> <span class="token string">"INGRESS"</span>\n<span class="token punctuation">}</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Application Layer: HTTP, gRPC, DNS, FQDN Policies"),"\n",t.createElement(e.p,null,"This is where Cilium visibly exceeds. Cilium extends network policy enforcement to Layers 3, 4, and 7, enabling more granular control over ingress and egress traffic based on application behavior, including DNS support and mTLS."),"\n",t.createElement(e.p,null,"The following are the common challenges in the application layer."),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"Misconfigured Network Policies are a common issue"),"\n",t.createElement(e.li,null,"DNS resolution failure"),"\n"),"\n",t.createElement(e.p,null,"Symptoms"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"HTTP 403/404 responses that only affect specific paths, not the whole service"),"\n",t.createElement(e.li,null,"DNS resolution fails for external FQDNs, but pod IPs are reachable"),"\n",t.createElement(e.li,null,"gRPC calls time out on specific methods but not others"),"\n",t.createElement(e.li,null,"CiliumNetworkPolicy with HTTP rules is blocking traffic that should be allowed"),"\n"),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># List all endpoints with L7 policy</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg endpoint list</span>\nENDPOINT POLICY <span class="token punctuation">(</span>ingress<span class="token punctuation">)</span> POLICY <span class="token punctuation">(</span>egress<span class="token punctuation">)</span> IDENTITY LABELS <span class="token punctuation">(</span>source:key<span class="token punctuation">[</span><span class="token operator">=</span>value<span class="token punctuation">]</span><span class="token punctuation">)</span> IPv6 IPv4 STATUS\n ENFORCEMENT ENFORCEMENT\n<span class="token number">180</span> Disabled Disabled <span class="token number">10064</span> k8s:app<span class="token operator">=</span>frontend <span class="token number">10.244</span>.2.199 ready\n k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>default\n k8s:io.cilium.k8s.policy.cluster<span class="token operator">=</span>kind-kind\n k8s:io.cilium.k8s.policy.serviceaccount<span class="token operator">=</span>default\n k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>default\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Inspect FQDN cache (is the DNS entry resolved and cached?)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg fqdn cache list</span>\nEndpoint Source FQDN TTL ExpirationTime IPs\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch all HTTP flows cluster-wide (requires L7 visibility enabled)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --protocol http --follow</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch DNS queries and responses</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --protocol dns --follow</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Trace a specific pod\'s HTTP traffic</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --from-pod default/frontend --protocol http --follow</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Watch TCP specifically</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --protocol tcp --follow</span>\nJun <span class="token number">24</span> <span class="token number">23</span>:10:00.032: <span class="token number">172.18</span>.0.3:58624 <span class="token punctuation">(</span>host<span class="token punctuation">)</span> -<span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443 <span class="token punctuation">(</span>kube-apiserver<span class="token punctuation">)</span> to-network FORWARDED <span class="token punctuation">(</span>TCP Flags: ACK, PSH<span class="token punctuation">)</span>\nJun <span class="token number">24</span> <span class="token number">23</span>:10:00.286: <span class="token number">172.18</span>.0.3:58636 <span class="token punctuation">(</span>host<span class="token punctuation">)</span> -<span class="token operator">></span> <span class="token number">172.18</span>.0.4:6443 <span class="token punctuation">(</span>kube-apiserver<span class="token punctuation">)</span> to-network FORWARDED <span class="token punctuation">(</span>TCP Flags: ACK, PSH<span class="token punctuation">)</span>\nJun <span class="token number">24</span> <span class="token number">23</span>:10:00.669: <span class="token number">10.244</span>.0.99:37982 <span class="token punctuation">(</span>world<span class="token punctuation">)</span> -<span class="token operator">></span> kube-system/hubble-relay-5bbf66658c-k6958:4245 <span class="token punctuation">(</span>ID:14331<span class="token punctuation">)</span> to-overlay FORWARDED <span class="token punctuation">(</span>TCP Flags: ACK, PSH<span class="token punctuation">)</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Show L7 policy verdicts (ALLOW vs DENY with path detail)</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --type policy-verdict --protocol http --follow</span>\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Cross-layer Challenges Common in Kubernetes"),"\n",t.createElement(e.p,null,"These challenges often happen when failures occur across different layers of the stack."),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"MTU Cascade: Incorrect MTU settings can result in packet drops and performance degradation. This is particularly difficult to diagnose, as it might only happen when transferring large payloads. To detect this in your cluster, prioritise using Hubble to monitor and analyse your network traffic. It is important to verify Cilium MTU configuration."),"\n",t.createElement(e.li,null,"Policy Drift: This occurs when the current state of your cluster configuration deviates from the defined state as specified in code templates. To detect this in your cluster, prioritise using Hubble to monitor."),"\n"),"\n",t.createElement(e.h1,null,"2.0 The Cilium Diagnostic Toolkit"),"\n",t.createElement(e.p,null,"Cilium provides a comprehensive set of built-in troubleshooting tools that leverage its eBPF data plane to expose information that is impossible to obtain with traditional networking tools. When a pod can't connect to a service, traceroute and tcpdump provide incomplete answers because they operate outside the kernel eBPF context where Cilium performs its work."),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Overall Healthcheck</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># cilium status --verbose</span>\n /¯¯<span class="token punctuation">\\</span>\n /¯¯<span class="token punctuation">\\</span>__/¯¯<span class="token punctuation">\\</span> Cilium: OK\n <span class="token punctuation">\\</span>__/¯¯<span class="token punctuation">\\</span>__/ Operator: OK\n /¯¯<span class="token punctuation">\\</span>__/¯¯<span class="token punctuation">\\</span> Envoy DaemonSet: OK\n <span class="token punctuation">\\</span>__/¯¯<span class="token punctuation">\\</span>__/ Hubble Relay: OK\n <span class="token punctuation">\\</span>__/ ClusterMesh: disabled\n\nDaemonSet cilium Desired: <span class="token number">3</span>, Ready: <span class="token number">3</span>/3, Available: <span class="token number">3</span>/3\nDaemonSet cilium-envoy Desired: <span class="token number">3</span>, Ready: <span class="token number">3</span>/3, Available: <span class="token number">3</span>/3\nDeployment cilium-operator Desired: <span class="token number">1</span>, Ready: <span class="token number">1</span>/1, Available: <span class="token number">1</span>/1\nDeployment hubble-relay Desired: <span class="token number">1</span>, Ready: <span class="token number">1</span>/1, Available: <span class="token number">1</span>/1\nDeployment hubble-ui Desired: <span class="token number">1</span>, Ready: <span class="token number">1</span>/1, Available: <span class="token number">1</span>/1\nContainers: cilium Running: <span class="token number">3</span>\n cilium-envoy Running: <span class="token number">3</span>\n cilium-operator Running: <span class="token number">1</span>\n clustermesh-apiserver\n hubble-relay Running: <span class="token number">1</span>\n hubble-ui Running: <span class="token number">1</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Check all Cilium pods</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get pods -n kube-system -l k8s-app=cilium</span>\nNAME READY STATUS RESTARTS AGE\ncilium-5pks9 <span class="token number">1</span>
1/1 Running <span class="token number">0</span> 8m50s\ncilium-rl6n5 <span class="token number">1</span>/1 Running <span class="token number">0</span> 8m50s\ncilium-wdvhp <span class="token number">1</span>/1 Running <span class="token number">0</span> 8m50s\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Perform endpoint inspection</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl exec -n kube-system cilium-6vxzs -- cilium-dbg endpoint list</span>\nENDPOINT POLICY <span class="token punctuation">(</span>ingress<span class="token punctuation">)</span> POLICY <span class="token punctuation">(</span>egress<span class="token punctuation">)</span> IDENTITY LABELS <span class="token punctuation">(</span>source:key<span class="token punctuation">[</span><span class="token operator">=</span>value<span class="token punctuation">]</span><span class="token punctuation">)</span> IPv6 IPv4 STATUS\n ENFORCEMENT ENFORCEMENT\n<span class="token number">126</span> Disabled Disabled <span class="token number">8</span> reserved:ingress <span class="token number">10.244</span>.0.14 ready\n<span class="token number">839</span> Disabled Disabled <span class="token number">4</span> reserved:health <span class="token number">10.244</span>.0.196 ready\n<span class="token number">2424</span> Disabled Disabled <span class="token number">1</span> k8s:node-role.kubernetes.io/control-plane ready\n k8s:node.kubernetes.io/exclude-from-external-load-balancers\n reserved:host\nroot@server:~<span class="token comment">#</span>\n\nroot@server:~<span class="token comment"># kubectl exec -n kube-system cilium-6vxzs -- cilium-dbg endpoint get 839</span>\n<span class="token punctuation">[</span>\n <span class="token punctuation">{</span>\n <span class="token string">"id"</span><span class="token builtin class-name">:</span> <span class="token number">839</span>,\n <span class="token string">"spec"</span><span class="token builtin class-name">:</span> <span class="token punctuation">{</span>\n <span class="token string">"label-configuration"</span><span class="token builtin class-name">:</span> <span class="token punctuation">{</span><span class="token punctuation">}</span>,\n <span class="token string">"options"</span><span class="token builtin class-name">:</span> <span class="token punctuation">{</span>\n <span class="token string">"ConntrackAccounting"</span><span class="token builtin class-name">:</span> <span class="token string">"Disabled"</span>,\n <span class="token string">"Debug"</span><span class="token builtin class-name">:</span> <span class="token string">"Disabled"</span>,\n <span class="token string">"DebugLB"</span><span class="token builtin class-name">:</span> <span class="token string">"Disabled"</span>,\n <span class="token string">"DebugPolicy"</span><span class="token builtin class-name">:</span> <span class="token string">"Disabled"</span>,\n <span class="token string">"DropNotification"</span><span class="token builtin class-name">:</span> <span class="token string">"Enabled"</span>,\n <span class="token string">"MonitorAggregationLevel"</span><span class="token builtin class-name">:</span> <span class="token string">"Medium"</span>,\n <span class="token string">"PolicyAccounting"</span><span class="token builtin class-name">:</span> <span class="token string">"Enabled"</span>
1,\n <span class="token string">"PolicyAuditMode"</span><span class="token builtin class-name">:</span> <span class="token string">"Disabled"</span>,\n <span class="token string">"PolicyVerdictNotification"</span><span class="token builtin class-name">:</span> <span class="token string">"Enabled"</span>,\n <span class="token string">"SourceIPVerification"</span><span class="token builtin class-name">:</span> <span class="token string">"Enabled"</span>,\n <span class="token string">"TraceNotification"</span><span class="token builtin class-name">:</span> <span class="token string">"Enabled"</span>\n <span class="token punctuation">}</span>\n <span class="token punctuation">}</span>,\n<span class="token punctuation">..</span>.\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Running Connectivity Test</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># cilium connectivity test</span>\nâ¹ï¸ Monitor aggregation detected, will skip some flow validation steps\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Creating namespace cilium-test-1 <span class="token keyword">for</span> connectivity check<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying echo-same-node service<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying DNS <span class="token builtin class-name">test</span> server configmap<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying same-node deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client2 deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying client3 deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying echo-other-node service<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying other-node deployment<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>host-netns<span class="token punctuation">]</span> Deploying kind-kind daemonset<span class="token punctuation">..</span>.\n⨠<span class="token punctuation">[</span>host-netns-non-cilium<span class="token punctuation">]</span> Deploying kind-kind daemonset<span class="token punctuation">..</span>.\nâ¹ï¸ Skipping tests that require a <span class="token function">node</span> Without Cilium\n⨠<span class="token punctuation">[</span>kind-kind<span class="token punctuation">]</span> Deploying Ingress resource<span class="token punctuation">..</span>.\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/cb0b585941d4a9c0f452a866cca9418f/74df6/decision-tree.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 101.58730158730158%; position: relative; bottom: 0; left: 0; background-image: url(\'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAACXBIWXMAABYlAAAWJQFJUiTwAAAD+klEQVR42pWUeUxcVRTGJ8YYlyZN3FKlaooVChQslaAVtzSp0dhKo221Wkss/NOgJVGsgG1EbYUS1BCif2gi0NKyD8uwDNsM2zDDPpQZhpbVgBTUSqUwMOvP+97DkNRG7E2+fOe8ufO9c+757lOxxvJ6vTJ7BJ86dZrY2FiSkpJITEyQOSYmmpycs8oejxsVt7C01RqK8s9TXVmOuriAmsoK8nJz6DS1rwh6ULmdThS4cLskOBV2KuxZqbDv0hRZ1b1caBigpMlKsf4iRTqLnGdV9TA1OyfvW7NCh3iRaJzU7DpUTx5l/UvHCXorhdD30vHfd5q7Ij7itvBjNPWMKILdGjWmknx6qsrpqa6gq1JNr1ZDe+EFzLVVokLRitvN3Lwd2/gMQxOzWEav0GUZwzIyLedDEzPYl52KYFzgo0RvuJvj2/1IejqIhDB/EsIDiLr3dhJ2BONWRsPY1O9UG6wY+kcxXhxH32WT45beEWrarfw5v6gI/mLpZ7ijnUvGVi4b27hsMohY4UmbBbdHOcOvfqpGFXKUx/YkE3zwDCHvphH0dioPvpyEavsH6LuHFcFxcw+j3Z1M2QaZtA4wOWgRsZWRLhPSy+T6hB3Gp6+KSgbRGm1oBVe1DchxrUCdySYqtCuCsU88zDvrVBzb4kN8yCYBX+ICHuHAHSrinwrAIw3G4fjXsJxOh3wUNy6Vub4Gk7pQoIDO8mI6y4roKC2UB2VtbvzH3TeYHZaXl2/qiv+0jcvlZmnlj6X6fiI//pHDn5/j08wKPskoJe6bEg5+ls3+hJ+xjl1RBCVLSAZ2CSNLcIr27IuLLC3ZsdsFlhTBzIImMYBE/N78krCodJ458h2hh9Lw3ZvMxt0nxeTHbl6hdH1cLtea11Bq2eV03bxly/gwWTWllDTXUd7WSHFTLRUGHfm6Gs7Vabh2fZ7hqQmytaWoW+rRGPSU6Gspa22gqElLbr2G2bmrq4LJ2d+jCn2IDfteJDjmDba8/zqBRyK559UwVDt8MY8Mkak+L/zmwwN7I+TfgqIj8Yt6jXXSnojN6Ho7VgUnZqZp6DHRZO4W3EG9iHV9XehFLvGCOM9f/5ihvrcdndlEY5+R2u42Gs1GOZcwtzC/Kuj1uvG6lwWcOJYWWLg+h8cl5Q4Zsk8k50hHJq6sRzxavLaE17HyzLVqSVnwh9wUfJ5VsW33/ew8FMCuw1uJOLCJra+sZ/POO8Xl76Ugqwv/+z7kucCTRL6Qxv5d37Ln+TOEP57Ito3xdBtHVgWHJ2xUNOShbVFTb6igwaChrrWMmuYSKnVFLC7/halllBNxeaR/oSEjRUtmai0ZX2tJPVFGcnwhE6O//b/v4a2uvwE+vy80QM+jIAAAAABJRU5ErkJggg==\'); background-size: cover; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/cb0b585941d4a9c0f452a866cca9418f/2ff5b/decision-tree.webp 252w,\n/static/cb0b585941d4a9c0f452a866cca9418f/4d583/decision-tree.webp 504w,\n/static/cb0b585941d4a9c0f452a866cca9418f/905a7/decision-tree.webp 1008w,\n/static/cb0b585941d4a9c0f452a866cca9418f/bb9f8/decision-tree.webp 1512w,\n/static/cb0b585941d4a9c0f452a866cca9418f/5d2fd/decision-tree.webp 1552w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/cb0b585941d4a9c0f452a866cca9418f/019e0/decision-tree.png 252w,\n/static/cb0b585941d4a9c0f452a866cca9418f/0dcb2/decision-tree.png 504w,\n/static/cb0b585941d4a9c0f452a866cca9418f/832a9/decision-tree.png 1008w,\n/static/cb0b585941d4a9c0f452a866cca9418f/19357/decision-tree.png 1512w,\n/static/cb0b585941d4a9c0f452a866cca9418f/74df6/decision-tree.png 1552w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/cb0b585941d4a9c0f452a866cca9418f/832a9/decision-tree.png"\n alt="Troubleshooting Decision Tree"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",t.createElement(e.em,null,"Figure 2: Cilium connectivity troubleshooting decision tree")),"\n",t.createElement(e.p,null,"Policy Troubleshooting"),"\n",t.createElement(e.p,null,"Cilium offers command-line utilities to interact with and troubleshoot network policies. The infrastru
1cture team can simulate the policy decision for a specific source or destination port combination and show exactly which rule is permitting or denying the flow, without sending real traffic."),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Get an effective policy for an endpoint</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg policy get</span>\n<span class="token punctuation">[</span><span class="token punctuation">]</span>\nRevision: <span class="token number">1</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"Log Inspection"),"\n",t.createElement(e.p,null,"Logs are the audit trail for everything the agent tried and failed to do. Cilium agent logs are verbose by default; the key is filtering for actionable signals rather than reading every line."),"\n",t.createElement(e.p,null,t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<code class="language-text"># Get Cilium logs</code>'}})),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system logs -f ds/cilium -c cilium-agent</span>\nFound <span class="token number">3</span> pods, using pod/cilium-6th6g\n<span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">2026</span>-06-27T14:00:11.756428436Z <span class="token assign-left variable">level</span><span class="token operator">=</span>info <span class="token assign-left variable">msg</span><span class="token operator">=</span><span class="token string">"Memory available for map entries (0.250% of 8322646016B): 20806615B"</span>\n<span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">2026</span>-06-27T14:00:11.756524087Z <span class="token assign-left variable">level</span><span class="token operator">=</span>info <span class="token assign-left variable">msg</span><span class="token operator">=</span><span class="token string">"option bpf-ct-global-tcp-max set by dynamic sizing to 131072"</span>\n<span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">2026</span>-06-27T14:00:11.756538881Z <span class="token assign-left variable">level</span><span class="token operator">=</span>info <span class="token assign-left variable">msg</span><span class="token operator">=</span><span class="token string">"option bpf-ct-global-any-max set by dynamic sizing to 65536"</span>\n<span class="token punctuation">..</span>.</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Endpoint regeneration log lines are particularly useful. Every time a pod's policy changes (new NetworkPolicy, label update, pod restart), Cilium regenerates the endpoint's eBPF programs. A regeneration failure means the pod's policy is stale."),"\n",t.createElement(e.h1,null,"3.0 Cluster Network State"),"\n",t.createElement(e.p,null,"One of the most dangerous failure modes in Kubernetes networking is the one that looks like nothing. Pods schedule, nodes appear Ready, ",t.createElement(e.strong,null,t.createElement(e.em,null,"kubectl get pods"))," shows green, but traffic silently fails, DNS resolutions time out, or cross-namespace calls drop without a clear error."),"\n",t.createElement(e.p,null,"Silence in Cilium can mean:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"A controller is reconciling but silently erroring. Controllers retry on failure and don't surface errors unless you explicitly check controller status. Run ",t.createElement(e.strong,null,t.createElement(e.em,null,"cilium-dbg status --verbose"))," and look for controllers with a non-zero failure count."),"\n",t.createElement(e.li,null,"Hubble flows are missing. If Hubble shows no flows for a pod that should be receiving traffic, the eBPF program may not be loaded on that node, or the endpoint hasn't been picked up by the agent. Check the cilium-dbg endpoint list and verify the pod's endpoint exists."),"\n",t.createElement(e.li,null,"eBPF programs aren't attached. The agent may be running but failing to attach programs to a new interface. This produces no obvious error in kubectl get pods, but traffic will bypass Cilium entirely. Verify with cilium-dbg bpf metrics list."),"\n",t.createElement(e.li,null,"KVStore lag. If you're using etcd as the KVStore, a lagging or partitioned etcd means node identities and policies are stale. Traffic may still flow based on cached state until the cache expires."),"\n"),"\n",t.createElement(e.h2,null,"Cilium Agent Health Check"),"\n",t.createElement(e.p,null,"The Cilium agent runs as a DaemonSet pod. A pod in Running state doesn't guarantee the agent is fully functional; it means the container started. True health requires checking what the agent reports about itself."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"Check agent status across all nodes.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system get pods -l k8s-app=cilium -o wide</span>\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\ncilium-9p44h <span class="token number">1</span>
1/1 Running <span class="token number">0</span> 86s <span class="token number">172.18</span>.0.4 kind-worker2 <span class="token operator"><</span>none<span class="token operator">></span> <span class="token operator"><</span>none<span class="token operator">></span>\ncilium-hlntk <span class="token number">1</span>/1 Running <span class="token number">0</span> 86s <span class="token number">172.18</span>.0.2 kind-worker <span class="token operator"><</span>none<span class="token operator">></span> <span class="token operator"><</span>none<span class="token operator">></span>\ncilium-jzkj4 <span class="token number">1</span>/1 Running <span class="token number">0</span> 86s <span class="token number">172.18</span>.0.3 kind-control-plane <span class="token operator"><</span>none<span class="token operator">></span> <span class="token operator"><</span>none<span class="token operator">></span>\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Look for OOMKill events, failed readiness probes, and init container failures. Cilium's init container (cilium-init) is responsible for setting up the host network before the main agent starts. Failures here mean the agent never reached a healthy state."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"Check the Cilium operator.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system get pods -l name=cilium-operator</span>\nNAME READY STATUS RESTARTS AGE\ncilium-operator-57f4d55695-gvhln <span class="token number">1</span>/1 Running <span class="token number">0</span> 5m19s\nroot@server:~<span class="token comment">#</span>\n\nroot@server:~<span class="token comment"># kubectl -n kube-system logs deployment/cilium-operator --tail=100</span>\n<span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">2026</span>-06-27T15:15:04.847030224Z <span class="token assign-left variable">level</span><span class="token operator">=</span>info <span class="token assign-left variable">msg</span><span class="token operator">=</span><span class="token string">"Start hook executed"</span> <span class="token assign-left variable">module</span><span class="token operator">=</span>operator <span class="token assign-left variable">function</span><span class="token operator">=</span>*job.groupHooks.Start <span class="token assign-left variable">duration</span><span class="token operator">=</span>645ns\n<span class="token assign-left variable">time</span><span class="token operator">=</span><span class="token number">2026</span>-06-27T15:15:04.847041107Z <span class="token assign-left variable">level</span><span class="token operator">=</span>info <span class="token assign-left variable">msg</span><span class="token operator">=</span><span class="token string">"Start hook executed"</span> <span class="token assign-left variable">module</span><span class="token operator">=</span>operator <span class="token assign-left variable">function</span><span class="token operator">=</span>*job.groupHooks.Start <span class="token assign-left variable">duration</span><span class="token operator">=</span>572ns</code></pre></div>'}}),"\n",t.createElement(e.p,null,"The operator manages cluster-scoped resources, CiliumNode objects, IPAM pools, and CRD reconciliation. An unhealthy operator can leave nodes unable to acquire IP addresses, even when agent pods themselves look fine."),"\n",t.createElement(e.h2,null,"Node Readiness"),"\n",t.createElement(e.p,null,"A node being Ready from Kubernetes' perspective and a node being fully operational from Cilium's perspective are different things. Kubernetes marks a node ready when its kubelet is healthy; Cilium readiness depends on the eBPF datapath being set up correctly."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get ciliumnode kind-worker2 -o yaml</span>\napiVersion: cilium.io/v2\nkind: CiliumNode\nmetadata:\n creationTimestamp: <span class="token string">"2026-06-27T15:15:03Z"</span>\n generation: <span class="token number">1</span>\n labels:\n beta.kubernetes.io/arch: amd64\n beta.kubernetes.io/os: linux\n kubernetes.io/arch: amd64\n kubernetes.io/hostname: kind-worker2\n kubernetes.io/os: linux\n<span class="token punctuation">..</span>.\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"The CiliumNode resource represents the node from Cilium's perspective. Its status section contains the IPAM state, including which IP addresses are allocated, which are available, and whether the node has successfully registered with the Cilium operator. A CiliumNode that hasn't been created, or whose spec diverges from the actual node's configuration, is a strong signal of an operator or RBAC problem."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"Verifying eBPF programs are loaded.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec cilium-jzkj4 -c cilium-agent -- bpftool prog list | head -n 5</span>\n<span class="token number">61</span>: cgroup_device tag 3918c82a5f4c0360\n loaded_at <span class="token number">2026</span>-06-27T13:19:14+0000 uid <span class="token number">0</span>\n xlated 64B jited 45B memlock 4096B\n<span class="token number">64</span>: cgroup_device tag 3918c82a5f4c0360\n loaded_at <span class="token number">2026</span>-06-27T13:19:14+0000 uid <span class="token number">0</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.h1,null,"4.0 Pod Connectivity Problems"),"\n",t.createElement(e.h2,null,"When the Pod Is Not Running"),"\n",t.createElement(e.p,null,"A pod that hasn't reached the running state has a networking problem that happens before networking c
1an be verified. Don't reach for Hubble yet; the issue is earlier in the lifecycle.\nStart with kubectl describe pod."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl describe pod nginx | head -n 5</span>\nName: nginx\nNamespace: default\nPriority: <span class="token number">0</span>\nService Account: default\nNode: kind-worker2/172.18.0.4\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"In the Events section, confirm the following details:"),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"FailedScheduling"),": the pod was never placed on a node. This is a scheduler problem, not a network problem. But if it's failing due to IPAM exhaustion, it will appear here."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"NetworkPluginNotReady"),": the CNI plugin (Cilium) hasn't signaled readiness to the kubelet yet. The Cilium DaemonSet pod on that node may be crashing or not yet fully initialized."),"\n",t.createElement(e.p,null,t.createElement(e.strong,null,"Failed to create pod sandbox"),": the container runtime couldn't set up the network namespace. This usually means the CNI binary isn't present at /opt/cni/bin/ or the CNI config at /etc/cni/net.d/ is malformed."),"\n",t.createElement(e.h2,null,"When the Pod Is Running but Unreachable"),"\n",t.createElement(e.p,null,"A running pod that doesn't respond to traffic is one of the most common Cilium troubleshooting scenarios. The most likely causes are: NetworkPolicy blocking the traffic, the pod's eBPF endpoint not being regenerated after a policy change, or the pod's IP not being correctly advertised."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"First: Confirm the pod has an endpoint.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec cilium-vnwrl -- cilium-dbg endpoint list | grep 10.244.2.57</span>\n<span class="token number">2043</span> Disabled Disabled <span class="token number">64004</span> k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>default <span class="token number">10.244</span>.2.57 ready\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"The endpoint should be in a ready state. If it's not-ready, regenerating, or disconnected, the datapath isn't set up for that pod yet. Give it 10 - 15 seconds; if it doesn't transition, check agent logs for regeneration errors."),"\n",t.createElement(e.h3,null,"Second: Check Hubble for dropped flows"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># hubble observe --to-pod default/nginx --verdict DROPPED --follow</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Then attempt to connect to the pod from a source. If drops appear in Hubble, the output will include the drop reason, typically ",t.createElement(e.strong,null,t.createElement(e.em,null,"Policy denied"))," or ",t.createElement(e.strong,null,t.createElement(e.em,null,"CT: missing entry")),". A Policy denied drop means a NetworkPolicy is blocking the traffic. An ",t.createElement(e.strong,null,t.createElement(e.em,null,"CT: missing entry"))," drop means the connection tracking table doesn't have a return-path entry, which suggests asymmetric routing."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Third: Trace the policy.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl exec -n kube-system ds/cilium -c cilium-agent -- cilium endpoint list</span>\nENDPOINT POLICY <span class="token punctuation">(</span>ingress<span class="token punctuation">)</span> POLICY <span class="token punctuation">(</span>egress<span class="token punctuation">)</span> IDENTITY LABELS <span class="token punctuation">(</span>source:key<span class="token punctuation">[</span><span class="token operator">=</span>value<span class="token punctuation">]</span><span class="token punctuation">)</span> IPv6 IPv4 STATUS\n ENFORCEMENT ENFORCEMENT\n<span class="token number">
1381</span> Disabled Disabled <span class="token number">54455</span> k8s:app<span class="token operator">=</span>client <span class="token number">10.244</span>.1.83 ready\n k8s:io.cilium.k8s.namespace.labels.kubernetes.io/metadata.name<span class="token operator">=</span>default\n k8s:io.cilium.k8s.policy.cluster<span class="token operator">=</span>kind-kind\n k8s:io.cilium.k8s.policy.serviceaccount<span class="token operator">=</span>default\n k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>default\n<span class="token punctuation">..</span><span class="token punctuation">..</span>\n\nroot@server:~<span class="token comment"># kubectl exec -n kube-system ds/cilium -c cilium-agent -- cilium policy get 1994</span>\n<span class="token punctuation">[</span><span class="token punctuation">]</span>\nRevision: <span class="token number">1</span>\nroot@server:~<span class="token comment">#</span>\n\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"The trace output shows you the exact policy decision. If no ingress rule matches, Cilium will tell you. This is far faster than reading NetworkPolicy YAML and mentally evaluating label selectors."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Fourth: Verify Service routing"),"\nIf you're hitting the pod through a Service, itâs important to check the BPF service map."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># CILIUM_AGENT_POD=$(kubectl -n kube-system get pods -l k8s-app=cilium -o jsonpath=\'{.items[0].metadata.name}\')</span>\n\nkubectl <span class="token parameter variable">-n</span> kube-system <span class="token builtin class-name">exec</span> <span class="token variable">$CILIUM_AGENT_POD</span> -- cilium-dbg bpf lb list <span class="token operator">|</span> <span class="token function">grep</span> <span class="token number">10.96</span>.194.145\n<span class="token number">10.96</span>.194.145:80/TCP <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">20</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\n<span class="token number">10.96</span>.194.145:0/ANY <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"A Service that has no backend entries in the BPF load balancer table will silently drop traffic even if the pod itself is healthy."),"\n",t.createElement(e.h2,null,"Cross-Node Pod Connectivity"),"\n",t.createElement(e.p,null,"Cross-node failures are network-topology failures. When a pod on ",t.createElement(e.strong,null,"Node A")," can't reach a pod on ",t.createElement(e.strong,null,"Node B"),", the problem is usually in one of three layers: the underlay (physical or cloud routing), the Cilium tunnel or native routing configuration, or the eBPF programs on the receiving node."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -c cilium-agent -- cilium-dbg status --verbose | grep -E "Routing|Tunnel"</sp
1an>\nKubeProxyReplacement: True <span class="token punctuation">[</span>eth0 <span class="token number">172.18</span>.0.2 fc00:f853:ccd:e793::2 fe80::b4cd:bdff:fe8f:95db <span class="token punctuation">(</span>Direct Routing<span class="token punctuation">)</span><span class="token punctuation">]</span>\nRouting: Network: Tunnel <span class="token punctuation">[</span>vxlan<span class="token punctuation">]</span> Host: Legacy\n Devices: eth0 <span class="token number">172.18</span>.0.2 fc00:f853:ccd:e793::2 fe80::b4cd:bdff:fe8f:95db <span class="token punctuation">(</span>Direct Routing<span class="token punctuation">)</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"Cilium supports two modes:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,t.createElement(e.strong,null,"VXLAN / Geneve (tunnel mode);")," pod traffic is encapsulated between nodes. Node-to-Node UDP connectivity on port 8472 (VXLAN) or 6081 (Geneve) must be allowed."),"\n",t.createElement(e.li,null,t.createElement(e.strong,null,"Native routing:")," pod CIDR routes must be present in the underlay (via BGP, cloud routing tables, or static routes). No encapsulation means any routing gap in the underlay creates a black hole."),"\n"),"\n",t.createElement(e.p,null,"A missing route means the pod CIDR for the destination node hasn't been advertised. In BGP mode, check that the Cilium BGP control plane has established a session and is advertising prefixes:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">kubectl <span class="token parameter variable">-n</span> kube-system <span class="token builtin class-name">exec</span> <span class="token operator"><</span>cilium-pod<span class="token operator">></span> -- cilium-dbg bgp peers\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Running Connectivity Tests with Cilium"),"\n",t.createElement(e.p,null,"Rather than constructing test traffic manually, Cilium includes a built-in connectivity test suite that validates the entire network stack end-to-end. It's the fastest way to confirm that a cluster's networking is fully operational after installation, an upgrade, or a configuration change."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Run the full connectivity test.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">cilium connectivity <span class="token builtin class-name">test</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"This deploys a set of test pods across your cluster and runs a structured battery of tests, including:"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"Pod-to-pod connectivity within the same node"),"\n",t.createElement(e.li,null,"Pod-to-pod connectivity across nodes"),"\n",t.createElement(e.li,null,"Pod-to-Service connectivity"),"\n",t.createElement(e.li,null,"Pod-to-external connectivity (egress)"),"\n",t.createElement(e.li,null,"NetworkPolicy enforcement (allow and deny cases)"),"\n",t.createElement(e.li,null,"DNS resolution"),"\n",t.createElement(e.li,null,"L7 policy enforcement (if Hubble and Envoy are enabled)"),"\n"),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Run a subset of tests.")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell"><span class="token comment"># Only pod-to-pod tests</span>\ncilium connectivity <span class="token builtin class-name">test</span> <span class="token parameter variable">--test</span> pod-to-pod\n\n<span class="token comment"># Exclude tests requiring external access</span>\ncilium connectivity <span class="token builtin class-name">test</span> <span class="token parameter variable">--test</span> <span class="token string">
1\'!external-1.1.1.1\'</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"The test name tells you the scenario. The flow line tells you the exact src/dst. From here, use ",t.createElement(e.strong,null,t.createElement(e.em,null,"hubble observe --verdict DROPPED"))," filtered to those pods to identify the drop cause."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Use connectivity tests as a regression gate.")),"\n",t.createElement(e.p,null,"After any change to NetworkPolicy, Cilium configuration, or cluster topology, run the connectivity tests before marking the change complete. A clean pass means the full network stack datapath, policy engine, identity system, DNS, and Service load balancing are working as expected across all nodes."),"\n",t.createElement(e.h1,null,"5.0 Service and Load Balancing Connectivity Problems"),"\n",t.createElement(e.p,null,"Service failures in Kubernetes are deceptive. The pod may be healthy, the deployment may show all replicas ready, and yet traffic never arrives. With Ciliumâs kube-proxy replacement, service load balancing runs entirely in the eBPF datapath, no iptables rules, no kube-proxy chains. That means the debugging path is different and faster than with traditional CNIs."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://cilium.io/blog/2026/04/25/understanding-kubernetes-load-balancing/"},"Read More About Service Load Balancing")),"\n",t.createElement(e.h2,null,"Is the Service Selecting Any Pod?"),"\n",t.createElement(e.p,null,"The first question when a Service doesn't work is whether it has any backends at all. A Service with no matching pods will accept connections and immediately drop them silently, from the caller's perspective."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get endpoints nginx-service</span>\nWarning: v1 Endpoints is deprecated <span class="token keyword">in</span> v1.33+<span class="token punctuation">;</span> use discovery.k8s.io/v1 EndpointSlice\nNAME ENDPOINTS AGE\nnginx-service <span class="token number">10.244</span>.1.112:80 17s\nroot@server:~<span class="token comment">#</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"If the ",t.createElement(e.strong,null,t.createElement(e.em,null,"ENDPOINTS"))," column is empty or shows ",t.createElement(e.strong,null,t.createElement(e.em,null,"<none>")),", the selector isn't matching any pods. Cross-check the Service's ",t.createElement(e.strong,null,t.createElement(e.em,null,"spec. selector"))," against actual pod labels."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get svc nginx-service -o jsonpath=\'{.spec.selector}\'</span>\n<span class="token punctuation">{</span><span class="token string">"run"</span><span class="token builtin class-name">:</span><span class="token string">"nginx"</span><span class="token punctuation">}</span>\nroot@server:~<span class="token comment">#</span>\n\nroot@server:~<span class="token comment"># kubectl get pods --show-labels</span>\nNAME READY STATUS RESTARTS AGE LABELS\nnginx <span class="token number">1</span>/1 Running <span class="token number">0</span> 2m19s <span class="token assign-left variable">run</span><span class="token operator">=</span>nginx\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"A single mismatched label, a typo, a version suffix, or a missing key is enough to produce zero backends. The Service exists, the pods exist, but they never find each other."),"\n",t.createElement(e.p,null,"Also, verify pods are actually running and passing readiness probes. A pod in Running but failing its readiness check will be excluded from EndpointSlices automatically."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get pods --show-labels</span>\nNAME READY STATUS RESTARTS AGE LABELS\nnginx <span class="token number">1</span>
1/1 Running <span class="token number">0</span> 2m19s <span class="token assign-left variable">run</span><span class="token operator">=</span>nginx\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"ClusterIP Not Responding"),"\n",t.createElement(e.p,null,"You can validate the correct datapath state by running cilium-dbg bpf lb list to inspect the contents of the eBPF load balancing maps."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/operations/troubleshooting/"},"Read More About Cilium Troubleshooting")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg bpf lb list | grep 10.96.69.57</span>\n<span class="token number">10.96</span>.69.57:80/TCP <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span> <span class="token number">10.244</span>.1.112:80/TCP <span class="token punctuation">(</span><span class="token number">20</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span>\n<span class="token number">10.96</span>.69.57:80/TCP <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">20</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\n<span class="token number">10.96</span>.69.57:0/ANY <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token number">0.0</span>.0.0:0 <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">[</span>ClusterIP, non-routable<span class="token punctuation">]</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"If the ClusterIP appears in the table with no backends listed, the endpoint hasn't been programmed into the BPF map. Check the Cilium operator logs and verify EndpointSlices are in sync. If the ClusterIP doesn't appear at all, the Service hasn't been picked up by Cilium's service reconciler."),"\n",t.createElement(e.p,null,"Run ",t.createElement(e.strong,null,t.createElement(e.em,null,"cilium-dbg debuginfo"))," and look for the section ",t.createElement(e.strong,null,t.createElement(e.em,null,"k8s-service-cache")),". In that section, you will find the contents of the service correlation cache. This cache is what Cilium uses to reconcile Kubernetes Services into BPF map entries. A stale or missing entry here explains why the datapath isn't programmed correctly."),"\n",t.createElement(e.h2,null,"NodePort Not Reachable Externally"),"\n",t.createElement(e.p,null,"With the help of the cilium-dbg service list command, you can validate that Cilium's eBPF kube-proxy replacement created the NodePort service. Services with the NodePort will have entries for each device on every node."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/"},"Read More About Kube Proxy Replacement")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg service list | grep NodePort</span>\n\n<span class="token number">1</span> <span class="token number">0.0</span>
1.0.0:31234/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.2.211:4245/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">4</span> <span class="token number">0.0</span>.0.0:31235/TCP NodePort <span class="token number">1</span> <span class="token operator">=</span><span class="token operator">></span> <span class="token number">10.244</span>.2.123:8081/TCP <span class="token punctuation">(</span>active<span class="token punctuation">)</span>\n<span class="token number">12</span> <span class="token number">0.0</span>.0.0:30452/TCP NodePort\n<span class="token number">14</span> <span class="token number">0.0</span>.0.0:32142/TCP NodePort\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"If NodePort entries are missing for a node interface, the --devices configuration may not include that interface. Check the Cilium ConfigMap:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system get cm cilium-config -o jsonpath=\'{.data.devices}\'</span>\neth0\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"The device list must include the interface on which external traffic arrives. If your nodes have multiple network interfaces (common in cloud environments), an interface not listed here will not have NodePort BPF programs attached, and external traffic arriving there will be ignored.\nFor external traffic specifically, itâs important to verify that your cloud providerâs security groups or firewall rules allow the NodePort default range (30000-32767 TCP/UDP) from external sources."),"\n",t.createElement(e.p,null,"If the container runtime in your cluster is running in cgroup namespace mode, Cilium's kube-proxy replacement-based load balancing may not be effective, leading to connectivity issues. Verify cgroup attachment."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- bpftool cgroup tree /run/cilium/cgroupv2/</span>\nCgroupPath\nID AttachType AttachFlags Name\n/run/cilium/cgroupv2\n<span class="token number">734</span> cgroup_inet_sock_release multi cil_sock_release\n<span class="token number">731</span> cgroup_inet4_post_bind multi cil_sock4_post_bind\n<span class="token number">735</span> cgroup_inet6_post_bind multi cil_sock6_post_bind\n<span class="token number">745</span> cgroup_inet4_connect multi cil_sock4_connect\n<span class="token punctuation">..</span>.\n\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Debugging EndpointSlices"),"\n",t.createElement(e.p,null,"Cilium consumes EndpointSlices directly; it doesn't rely on the older Endpoints API. If EndpointSlices are missing, stale, or out of sync, the BPF load balancing table will not reflect the current backend state."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell"><span class="token comment"># List EndpointSlices for a service</span>\nroot@server:~<span class="token comment"># kubectl get endpointslices -l kubernetes.io/service-name=nginx-service</span>\nNAME ADDRESSTYPE PORTS ENDPOINTS AGE\nnginx-service-krzb2 IPv4 <span class="token number">80</span> <span class="token number">10.244</span>.1.112 15m\nroot@server:~<span class="token comment">#</span>\n\n<span class="token comment"># Inspect a specific slice</span>\nroot@server:~<span class="token comment"># kubectl describe endpointslice nginx-service-krzb2</span>\nName: nginx-service-krzb2\nNamespace: default\nLabels: endpointslice.kubernetes.io/managed-by<span class="token operator">=</span>endpointslice-controller.k8s.io\n kubernetes.io/service-name<span class="token operator">=</span>
1nginx-service\n <span class="token assign-left variable">run</span><span class="token operator">=</span>nginx\nAnnotations: endpoints.kubernetes.io/last-change-trigger-time: <span class="token number">2026</span>-07-19T12:25:21Z\nAddressType: IPv4\nPorts:\n Name Port Protocol\n ---- ---- --------\n <span class="token operator"><</span>unset<span class="token operator">></span> <span class="token number">80</span> TCP\nEndpoints:\n - Addresses: <span class="token number">10.244</span>.1.112\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"The following details must be present in the output"),"\n",t.createElement(e.ul,null,"\n",t.createElement(e.li,null,"addressType: should be IPv4 or IPv6 for dual stack."),"\n",t.createElement(e.li,null,"conditions. ready: only ready endpoints will be picked up."),"\n",t.createElement(e.li,null,"targetRef: should point to an existing pod in the expected namespace."),"\n"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl get endpointslices -l kubernetes.io/service-name=nginx-service -o jsonpath=\'{range .items[*]}{"addressType: "}{.addressType}{"\\n"}{range .endpoints[*]}{" - targetRef: "}{.targetRef.kind}{"/"}{.targetRef.name}{"\\n ready: "}{.conditions.ready}{"\\n"}{end}{end}\'</span>\naddressType: IPv4\n - targetRef: Pod/nginx\n ready: <span class="token boolean">true</span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.h2,null,"The Load Balancer Service Is Stuck in Pending"),"\n",t.createElement(e.p,null,"A LoadBalancer Service showing ",t.createElement(e.em,null,"<pending>")," in the EXTERNAL-IP column means no external IP has been assigned. The cause depends on your environment."),"\n",t.createElement(e.p,null,"On bare metal or on-premise clusters using Cilium LB IPAM:\nThe ExternalIP field having a value of ",t.createElement(e.em,null,"<pending>")," means no LB IPs have been assigned. When LB IPAM is unable to allocate or assign IPs for the service, it will update the service conditions in the status."),"\n",t.createElement(e.p,null,"To check for service status conditions:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl describe svc nginx-service</span>\nName: nginx-service\nNamespace: default\nLabels: <span class="token assign-left variable">run</span><span class="token operator">=</span>nginx\nAnnotations: <span class="token operator"><</span>none<span class="token operator">></span>\nSelector: <span class="token assign-left variable">run</span><span class="token operator">=</span>nginx\nType: ClusterIP\nIP Family Policy: SingleStack\nIP Families: IPv4\nIP: <span class="token number">10.96</span>.69.57\nIPs: <span class="token number">10.96</span>.69.57\nPort: <span class="token operator"><</span>unset<span class="token operator">></span> <span class="token number">80</span>/TCP\nTargetPort: <span class="token number">80</span>/TCP\nEndpoints: <span class="token number">10.244</span>.1.112:80\nSession Affinity: None\nInternal Traffic Policy: Cluster\nEvents: <span class="token operator"><</span>none<span class="token operator">></span>\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"A reason for no_pool means no ",t.createElement(e.strong,null,t.createElement(e.em,null,"CiliumLoadBalancerIPPool"))," has been created, or the pool's ",t.createElement(e.strong,null,t.createElement(e.em,null,"serviceSelector"))," doesn't match this service.\nLB IPAM is always enabled but dormant. The controller is only activated
1when the first IP Pool is added to the cluster. Create a pool if none exists"),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/network/lb-ipam/"},"Read More About LB IPAM")),"\n",t.createElement(e.h2,null,"Session Affinity"),"\n",t.createElement(e.p,null,"Session affinity ensures that repeated connections from the same client are routed to the same backend pod. Without it, stateful applications or anything maintaining in-memory session state will behave erratically when connections are distributed across replicas."),"\n",t.createElement(e.p,null,"Cilium implements session affinity natively in eBPF. The source for the affinity depends on the origin of a request. If a request is sent from outside the cluster to the service, the request's source IP address is used for determining the endpoint affinity. If a request is sent from inside the cluster, the client's network namespace cookie is used as the source when socket-LB is in use. This allows affinity to be implemented at the socket layer, where the endpoint selection happens before a network packet has been built."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#session-affinity"},"Read More About Cilium Session Affinity")),"\n",t.createElement(e.h1,null,"6.0 DNS Resolution Failures"),"\n",t.createElement(e.p,null,"DNS is a common cause for outages and incidents in Kubernetes clusters. When errors occur during DNS resolution, they are typically referred to as DNS resolution errors or DNS issues. Errors during connection to the returned IP are generally referred to as network connectivity issues. Keeping that distinction sharp saves significant debugging time; the symptoms can look identical at the application layer."),"\n",t.createElement(e.h2,null,"Running DNS Resolution Tests inside Pods"),"\n",t.createElement(e.p,null,"When DNS issues occur, they often go unnoticed until significant damage has been done."),"\n",t.createElement(e.p,null,"The first ideal step is to check whether nslookup is working properly."),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl run dnsutils --image=gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 --command -- sleep 3600</span>\npod/dnsutils created\nroot@server:~<span class="token comment"># kubectl exec -i -t dnsutils -- nslookup kubernetes.default</span>\nServer: <span class="token number">10.96</span>.0.10\nAddress: <span class="token number">10.96</span>.0.10<span class="token comment">#53</span>\n\nName: kubernetes.default.svc.cluster.local\nAddress: <span class="token number">10.96</span>.0.1\n\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"The following are common issues associated with DNS in Kubernetes"),"\n",t.createElement(e.ol,null,"\n",t.createElement(e.li,null,"Delayed response: the DNS server is slow to respond due to congestion or server overload."),"\n",t.createElement(e.li,null,"Incorrect DNS resolutions: usually because of out-of-date or incomplete records."),"\n",t.createElement(e.li,null,"DNS records conflict: multiple pods or services use the same name or IP."),"\n",t.createElement(e.li,null,"DNS lookup failures: When a lookup fails, it is most often due to the server being unreachable."),"\n",t.createElement(e.li,null,"NXDOMAIN errors: the server can't find a request domain within its records."),"\n",t.createElement(e.li,null,"External domain failures: Kubernetes DNS canât resolve the external domain name, likely due to failure to connect to the external DNS server."),"\n"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<span\n class="gatsby-resp-image-wrapper"\n style="position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 1008px; "\n >\n <a\n class="gatsby-resp-image-link"\n href="/static/4f014837761d83e70872528bf5d63315/44899/dns-queries.png"\n style="display: block"\n target="_blank"\n rel="noopener"\n >\n <span\n class="gatsby-resp-image-background-image"\n style="padding-bottom: 47.61904761904761%; position: relative; bottom: 0; left: 0; background-image: url(\'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAAAsTAAALEwEAmpwYAAABp0lEQVR42oVSO0sDQRA+EEEk3iPeXu7yuHCvXC6
15RKOCUQIRXxCiCFpqZ2OhhYWNVlYW9oKN/gsLGxu18j99zu7lpYlYfOzszs7M982MpGRC9KGa5TFb/fU+6e/oXeKGakbQrAoSuw/+lpwJymPQrEggsZM3kVDWHaRUC4rhQ6YqslES1ebotL0FhNVlugeEsvBx8LgZ1cH0nP2Tocw8sGAbxdYVtHyDJJYwn6siTZCZi2x8gGjrGooZE4sQ84W68Kd0H832Hnb3j6FlKwPpkkysGicv6N4D3uYtdNPBYrODIF4XSTYvPnD2BNhrlzAs8sVr8IIVTKUKeHh8xuvbFxWPiWEo5Etcrtu+wcrpJ6zaEdKmD0YsWKFG0gLUu3fYOH+HFXXIF4DlKtCJ4Wzaw/LqDtpbh8Q2GDLk2tOGDcPMQTMc0Sel10MOZrmw8i60jCd8vK9ccqnegltehe0vISQ76zZGEmYjqjqc2HBqEQVXwPI/t4D/0/MxnKgJn1qTKdYHsSKhIiY0eS0E494UBXqBYgtYIOTKxtAv/bfYyh+LrYwUHl2bb6/jGOdZTAEWAAAAAElFTkSuQmCC\'); background-size: cover; display: block;"\n ></span>\n <picture>\n <source\n srcset="/static/4f014837761d83e70872528bf5d63315/2ff5b/dns-queries.webp 252w,\n/static/4f014837761d83e70872528bf5d63315/4d583/dns-queries.webp 504w,\n/static/4f014837761d83e70872528bf5d63315/905a7/dns-queries.webp 1008w,\n/static/4f014837761d83e70872528bf5d63315/bb9f8/dns-queries.webp 1512w,\n/static/4f014837761d83e70872528bf5d63315/5aae5/dns-queries.webp 1999w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/webp"\n />\n <source\n srcset="/static/4f014837761d83e70872528bf5d63315/019e0/dns-queries.png 252w,\n/static/4f014837761d83e70872528bf5d63315/0dcb2/dns-queries.png 504w,\n/static/4f014837761d83e70872528bf5d63315/832a9/dns-queries.png 1008w,\n/static/4f014837761d83e70872528bf5d63315/19357/dns-queries.png 1512w,\n/static/4f014837761d83e70872528bf5d63315/44899/dns-queries.png 1999w"\n sizes="(max-width: 1008px) 100vw, 1008px"\n type="image/png"\n />\n <img\n class="gatsby-resp-image-image"\n src="/static/4f014837761d83e70872528bf5d63315/832a9/dns-queries.png"\n alt="DNS Queries"\n title=""\n loading="lazy"\n decoding="async"\n style="width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;"\n />\n </picture>\n </a>\n </span>'}}),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Figure3:DNS resolution queries")),"\n",t.createElement(e.h2,null,"Checking DNS Logs"),"\n",t.createElement(e.p,null,"Logs can be checked using the following command:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">kubectl logs <span class="token parameter variable">--namespace</span><span class="token operator">=</span>kube-system <span class="token parameter variable">-l</span> k8s-app<span class="token operator">=</span>kube-dns</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Warnings or error messages in the logs may help you hone in on the source of your DNS problem, especially if the root of the issue lies with kube-dns itself as opposed to the network."),"\n",t.createElement(e.h2,null,"Check kube-dns Resource Utilization"),"\n",t.createElement(e.p,null,"Checking the CPU utilization of the kube-DNS service can help you determine whether the server is simply overloaded. To view real-time CPU metrics, run:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl logs --namespace=kube-system -l k8s-app=kube-dns</span>\nmaxprocs: Leaving <span class="token assign-left variable">GOMAXPROCS</span><span class="token operator">=</span><span class="token number">8</span>: CPU <span class="token function">quota</span> undefined\n.:53\n<span class="token punctuation">[</span>INFO<span class="token punctuation">]</span> plugin/reload: Running configuration SHA512 <span class="token operator">=</span> 61c4e97fd46a540ac8dda919cc898bd4e91207731b7b08b141e56fbd76fd591d82a4524df658474b8e3d867076fc7ba04e95992926333c1bcfaa75c46969808c\nCoreDNS-1.12.0\nlinux/amd64, go1.23.3, 51e11f1\nmaxprocs: Leaving <span class="token assign-left variable">GOMAXPROCS</span><span class="token operator">=</span><span class="token number">8</span>: CPU <span class="token function">quota</span> undefined\n.:53\n<span class="token punctuation">[</span>INFO<span class="token punctuation">]</span>
1 plugin/reload: Running configuration SHA512 <span class="token operator">=</span> 61c4e97fd46a540ac8dda919cc898bd4e91207731b7b08b141e56fbd76fd591d82a4524df658474b8e3d867076fc7ba04e95992926333c1bcfaa75c46969808c\nCoreDNS-1.12.0\nlinux/amd64, go1.23.3, 51e11f1\nroot@server:~<span class="token comment">#</span>\n</code></pre></div>'}}),"\n",t.createElement(e.h1,null,"7.0 Network Policy Drops"),"\n",t.createElement(e.p,null,"Network policy changes are among the most frequent operations in a Kubernetes cluster. They are also among the most delicate, as even a small mistake can lead to widespread traffic disruption. This section walks through several methods to make policy management safer, especially in Day 2 operations or brownfield deployments where clusters already run critical workloads. It shows how to test and validate changes before enforcing them, helping teams adopt a more reliable approach to policy rollout."),"\n",t.createElement(e.p,null,"An endpoint can be put into default deny mode at ingress or egress if a rule selects the endpoint and contains the respective rule section. Any rule selecting the endpoint will have this effect. Understanding that default-deny is triggered implicitly, not by a dedicated deny rule, is the foundation of everything that follows."),"\n",t.createElement(e.p,null,"Cilium network policies build upon Kubernetes NetworkPolicy, extending it with deeper visibility and more flexible rule types. Policies are used to allow or deny traffic based on defined rules, which can apply to ingress, egress, or both. These rules are evaluated at the datapath level, meaning enforcement takes place directly in the kernel through eBPF."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://www.cncf.io/blog/2025/11/06/safely-managing-cilium-network-policies-in-kubernetes-testing-and-simulation-techniques/"},"Read More About Safely Managing Cilium Network Policies")),"\n",t.createElement(e.h2,null,"Identifying Policy-Denied Drops"),"\n",t.createElement(e.p,null,"The fastest way to identify a policy drop is through Hubble. Unlike iptables-based CNIs, where you parse iptables -L output and mentally trace rule ordering, Cilium surfaces every policy decision as an observable flow event, including the direction and reason for each drop."),"\n",t.createElement(e.p,null,t.createElement(e.em,null,"Watch for drops in real time:")),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell"><span class="token comment"># All policy-denied drops in the cluster</span>\nroot@server:~<span class="token comment"># hubble observe --verdict DROPPED --follow</span>\n\n\n<span class="token comment"># Filter to a specific destination</span>\nhubble observe <span class="token parameter variable">--verdict</span> DROPPED <span class="token punctuation">\\</span>\n --to-pod default/backend <span class="token parameter variable">--follow</span>\n\n<span class="token comment"># Filter by source pod</span>\nroot@server:~<span class="token comment"># hubble observe --verdict DROPPED \\</span>\n --to-pod default/nginx <span class="token parameter variable">--follow</span>\n\n<span class="token comment"># Show only policy-verdict events (broader than just drops)</span>\nroot@server:~<span class="token comment"># hubble observe -t policy-verdict --follow</span>\n\n</code></pre></div>'}}),"\n",t.createElement(e.p,null,"Using cilium-dbg monitor for low-level drop events:\nWhen Hubble is not available, or you need lower-level detail:"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg monitor --type drop</span>\nListening <span class="token keyword">for</span> events on <span class="token number">8</span>
1 CPUs with 64x4096 of shared memory\nPress Ctrl-C to quit\n\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Policy Tracing"),"\n",t.createElement(e.p,null,'Policy tracing simulates the policy decision for a specific traffic flow without sending any real traffic. It\'s the most direct way to answer the question: "Should this traffic be allowed right now, given the policies in effect?"'),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell"><span class="token comment"># Trace by pod name</span>\nkubectl <span class="token parameter variable">-n</span> kube-system <span class="token builtin class-name">exec</span> ds/cilium -- <span class="token punctuation">\\</span>\n cilium-dbg policy trace <span class="token punctuation">\\</span>\n --src-k8s-pod default/frontend <span class="token punctuation">\\</span>\n --dst-k8s-pod default/backend <span class="token punctuation">\\</span>\n <span class="token parameter variable">--dport</span> <span class="token number">8080</span>/TCP\n\n<span class="token comment"># Trace by label selectors</span>\nkubectl <span class="token parameter variable">-n</span> kube-system <span class="token builtin class-name">exec</span> ds/cilium -- <span class="token punctuation">\\</span>\n cilium-dbg policy trace <span class="token punctuation">\\</span>\n --src-identity k8s:app<span class="token operator">=</span>frontend,k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>default <span class="token punctuation">\\</span>\n --dst-identity k8s:app<span class="token operator">=</span>backend,k8s:io.kubernetes.pod.namespace<span class="token operator">=</span>default <span class="token punctuation">\\</span>\n <span class="token parameter variable">--dport</span> <span class="token number">8080</span>/TCP\n</code></pre></div>'}}),"\n",t.createElement(e.h2,null,"Rendering the Effective Policy"),"\n",t.createElement(e.p,null,"The policy in effect for an endpoint is not simply the sum of kubectl get networkpolicy output. Cilium computes a realized policy per endpoint by evaluating all matching rules from Kubernetes NetworkPolicies, CiliumNetworkPolicies, and CiliumClusterwideNetworkPolicies and merging them into a single enforcement state. The realized policy is what actually runs in the datapath."),"\n",t.createElement(e.p,null,"To determine which policy rules are currently in effect for an endpoint, the data from cilium-dbg endpoint list and cilium-dbg endpoint get can be paired with the data from cilium-dbg policy get. cilium-dbg endpoint get will list the labels of each rule that applies to an endpoint."),"\n",t.createElement(e.p,null,"Step 1: Find the endpoint ID"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># # 1. Find the Cilium pod handling your nginx pod</span>\n<span class="token assign-left variable">TARGET_NODE</span><span class="token operator">=</span><span class="token variable"><span class="token variable">$(</span>kubectl get pod nginx <span class="token parameter variable">-o</span> <span class="token assign-left variable">jsonpath</span><span class="token operator">=</span><span class="token string">\'{.spec.nodeName}\'</span><span class="token variable">)</span></span>\n<span class="token assign-left variable">CILIUM_POD</span><span class="token operator">=</span><span class="token variable"><span class="token variable">$(</span>kubectl <span class="token parameter variable">-n</span> kube-system get pods <span class="token parameter variable">-l</span> k8s-app<span class="token operator">=</span>cilium --field-selector <span class="token assign-left variable">spec.nodeName</span><span class="token operator">=</span>$TARGET_NODE <span class="token parameter variable">-o</span> <span class="token assign-left variable">jsonpath</span><span class="token operator">=</span><span class="token string">\'{.items[0].metadata.name}\'</span><span class="token variable">)</span></span>\n\n<span class="token comment"># 2. Extract the Endpoint ID</span>\n<span class="token assign-left variable">EP_ID</span><span class="token operator">=</span><span class="token variable"><span class="token variable">$(</span>kubectl <span class="token parameter variable">-n</span> kube-system <span class="token builtin class-name">exec</span>
1 $CILIUM_POD -- cilium-dbg endpoint list <span class="token operator">|</span> <span class="token function">grep</span> nginx <span class="token operator">|</span> <span class="token function">awk</span> <span class="token string">\'{print $1}\'</span><span class="token variable">)</span></span>\n\n<span class="token comment"># 3. Verify it found the ID</span>\n<span class="token builtin class-name">echo</span> <span class="token string">"Your Nginx Endpoint ID is: <span class="token variable">$EP_ID</span>"</span>\nYour Nginx Endpoint ID is: k8s:run<span class="token operator">=</span>nginx\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"Note the endpoint ID and the security identity from the output."),"\n",t.createElement(e.p,null,"Step 2: Check policy enforcement state"),"\n",t.createElement(e.span,{dangerouslySetInnerHTML:{__html:'<div class="gatsby-highlight" data-language="shell"><pre class="language-shell"><code class="language-shell">root@server:~<span class="token comment"># kubectl -n kube-system exec ds/cilium -- cilium-dbg endpoint list</span>\nENDPOINT POLICY <span class="token punctuation">(</span>ingress<span class="token punctuation">)</span> POLICY <span class="token punctuation">(</span>egress<span class="token punctuation">)</span> IDENTITY LABELS <span class="token punctuation">(</span>source:key<span class="token punctuation">[</span><span class="token operator">=</span>value<span class="token punctuation">]</span><span class="token punctuation">)</span> IPv6 IPv4 STATUS\n ENFORCEMENT ENFORCEMENT\n<span class="token number">98</span> Disabled Disabled <span class="token number">8</span> reserved:ingress <span class="token number">10.244</span>.0.12 ready\n<span class="token number">1050</span> Disabled Disabled <span class="token number">1</span> k8s:node-role.kubernetes.io/control-plane ready\n k8s:node.kubernetes.io/exclude-from-external-load-balancers\n reserved:host\n<span class="token number">2314</span> Disabled Disabled <span class="token number">4</span> reserved:health <span class="token number">10.244</span>.0.227 ready\nroot@server:~<span class="token comment">#</span></code></pre></div>'}}),"\n",t.createElement(e.p,null,"The POLICY (ingress) and POLICY (egress) columns show Enabled or Disabled for each endpoint. An endpoint with 'Disabled' on both means no policy is selecting it; it's in default-allow mode. The moment a policy is selected, the selected direction becomes the enforced default-deny. If you see Disabled when you expect Enabled, check whether the policy's endpointSelector actually matches the pod's labels."),"\n",t.createElement(e.h2,null,"Policymap Pressure"),"\n",t.createElement(e.p,null,"Each Cilium-managed endpoint has its own BPF policy map, a fixed-size hash table in kernel memory that holds all allowed traffic rules for that endpoint. The default maximum number of entries in the endpoint policy map is 16,384 per endpoint, configured via --bpf-policy-map-max.\nIn large clusters with complex policies, especially those using many CIDR rules, fine-grained label selectors, or toFQDNs, this map can fill up. When it does, policy entries can't be added, and traffic that should be allowed gets dropped."),"\n",t.createElement(e.p,null,"Common Pitfalls"),"\n",t.createElement(e.p,null,"1. Implicit default-deny on first policy application\nThe most common surprise in Cilium policy management: the moment you apply the first NetworkPolicy or CiliumNetworkPolicy that selects a pod, all traffic not explicitly permitted by that policy is denied. Applying the first network policy triggers the default-deny behavior, which may unintentionally block traffic not yet accounted for in your policy definitions.\nThe fix is to apply audit mode first, observe all traffic, then write policies that cover every observed flow before enforcing."),"\n",t.createElement(e.p,null,"2. Forgetting to allow egress DNS\nAny policy restricting egress must explicitly allow DNS on port 53. Without it, the pod loses service discovery. This is the single most common policy-related incident in production Cilium cluster
1s. Always include this egress rule in any policy applied to a namespace:"),"\n",t.createElement(e.p,null,"3. Policy is per-direction, not per-endpoint\nA policy that allows egress from Pod A to Pod B does not automatically allow ingress at Pod B. Note that while an egress rule may allow traffic from an endpoint, the receivers of that egress traffic may have ingress rules that deny it. Policy must be configured on both sides, the sender and the receiver. If both pods have policies applied, you need matching egress rules on the sender and ingress rules on the receiver."),"\n",t.createElement(e.p,null,"4. Label selector scope and namespace boundaries\nNetwork policies imported directly with the Cilium API apply to all namespaces unless a namespace selector is specified. A CiliumClusterwideNetworkPolicy without a namespace selector applies globally. A regular CiliumNetworkPolicy scopes to its own namespace. Getting this wrong produces either overly permissive or unexpectedly restrictive behavior. Always check the namespace context of your policy resource type."),"\n",t.createElement(e.p,null,"5. toServices and toPorts cannot be combined\nYou cannot create an egress rule that simultaneously specifies a service and restricts ports. You must choose between defining traffic by service identity or by port. Attempting to combine toServices with toPorts in the same egress rule produces a policy that silently ignores one of the two constraints, leading to unexpected behavior."),"\n",t.createElement(e.p,null,"6. The cluster entity does not span ClusterMesh\nA common misunderstanding when configuring entity-based L3 policies is that the cluster entity includes only endpoints in a local cluster and does not include other clusters in a ClusterMesh. If you're running ClusterMesh and your toEntities: cluster rule isn't allowing cross-cluster traffic, you need explicit selectors for the remote cluster's identities."),"\n",t.createElement(e.p,null,"7. toFQDNs rules without a DNS inspection rule\nAs covered in section 6.0, a toFQDNs rule does nothing without a companion L7 DNS rule that tells Cilium to intercept DNS responses. The traffic will be denied because Cilium never learned the IP mapping for the FQDN."),"\n",t.createElement(e.p,null,"8. Policy revision mismatch after rollout\nAfter applying a new policy, verify that the agent has picked it up on every node."),"\n",t.createElement(e.p,null,t.createElement(e.a,{href:"https://docs.cilium.io/en/stable/security/policy/troubleshooting/"},"Read More About Cilium Network Policy Troubleshooting")),"\n",t.createElement(e.h1,null,"8.0 Summary"),"\n",t.createElement(e.p,null,"Reactive troubleshooting, diagnosing a problem after it's already impacting production, is expensive. The gap between a network misconfiguration and its detection is often measured in minutes or hours, not seconds. The goal of this section is to close that gap by building a detection layer that surfaces issues before users notice them."),"\n",t.createElement(e.h4,null,"Troubleshooting Hierarchy"),"\n",t.createElement(e.p,null,"Every Cilium networking problem falls into one of four layers. Working top-down through this order eliminates entire categories of causes quickly"),"\n",t.createElement(e.table,null,t.createElement(e.thead,null,t.createElement(e.tr,null,t.createElement(e.th,{align:"left"},"Layer"),t.createElement(e.th,{align:"left"},"Check first"))),t.createElement(e.tbody,null,t.createElement(e.tr,null,t.createElement(e.td,{align:"left"},"Agent health"),t.createElement(e.td,{align:"left"},"cilium status all subsystems OK?")),t.createElement(e.tr,null,t.createElement(e.td,{align:"left"},"Endpoint state"),t.createElement(e.td,{align:"left"},"cilium-dbg endpoint list all endpoints ready?")),t.createElement(e.tr,null,t.createElement(e.td,{align:"left"},"Traffic visibility"),t.createElement(e.td,{align:"left"},"hubble observe --verdict DROPPED any drops?")),t.createElement(e.tr,null,t.createElement(e.td,{align:"left"},"Datapath state"),t.createElement(e.td,{align:"left"},"cilium-dbg bpf * maps reflect expected state?")))),"\n",t.createElement(e.p,null,"It is also important to utilise Prometheus metrics for alerting as an early warning on every major failure."),"\n",t.createElement(s,l.A.CharityMbisi))}e.A=function(n={}){const{wrapper:e}=Object.assign({},(0,a.RP)(),n.components);return e?t.createElement(e,n,t.createElement(o,n)):o(n)}}}]); 2//# sourceMappingURL=d09841dc-12b5b4fada72c5dd5c20.js.map
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.