1"use strict";(self.webpackChunktigera_docs=self.webpackChunktigera_docs||[]).push([[518],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>h});var a=n(67294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function p(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var l=a.createContext({}),s=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=s(e.components);return a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,l=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),d=s(n),h=i,f=d["".concat(l,".").concat(h)]||d[h]||u[h]||o;return n?a.createElement(f,r(r({ref:t},c),{},{components:n})):a.createElement(f,r({ref:t},c))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,r=new Array(o);r[0]=d;var p={};for(var l in t)hasOwnProperty.call(t,l)&&(p[l]=t[l]);p.originalType=e,p.mdxType="string"==typeof e?e:i,r[1]=p;for(var s=2;s<o;s++)r[s]=n[s];return a.createElement.apply(null,r)}return a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},3448:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>r,default:()=>u,frontMatter:()=>o,metadata:()=>p,toc:()=>s});var a=n(87462),i=(n(67294),n(3905));const o={description:"Specific troubleshooting steps for the VPP dataplane."},r="VPP dataplane troubleshooting",p={unversionedId:"operations/troubleshoot/vpp",id:"version-3.25/operations/troubleshoot/vpp",title:"VPP dataplane troubleshooting",description:"Specific troubleshooting steps for the VPP dataplane.",source:"@site/calico_versioned_docs/version-3.25/operations/troubleshoot/vpp.mdx",sourceDirName:"operations/troubleshoot",slug:"/operations/troubleshoot/vpp",permalink:"/calico/3.25/operations/troubleshoot/vpp",draft:!1,editUrl:"https://github.com/tigera/docs/edit/main/calico_versioned_docs/version-3.25/operations/troubleshoot/vpp.mdx",tags:[],version:"3.25",frontMatter:{description:"Specific troubleshooting steps for the VPP dataplane."},sidebar:"calicoSidebar",previous:{title:"Component logs",permalink:"/calico/3.25/operations/troubleshoot/component-logs"},next:{title:"Manage TLS certificates used by Calico",permalink:"/calico/3.25/operations/certificate-management"}},l={},s=[{value:"Big picture",id:"big-picture",level:2},{value:"Installing calivppctl",id:"installing-calivppctl",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"Kubernetes Cluster",id:"kubernetes-cluster",level:3},{value:"Starting calico-vpp-node Daemon set",id:"starting-calico-vpp-node-daemon-set",level:3},{value:"Having VPP up and running",id:"having-vpp-up-and-running",level:3},{value:"Accessing the VPP cli",id:"accessing-the-vpp-cli",level:2},{value:"Listing interfaces and basics",id:"listing-interfaces-and-basics",level:3},{value:"Tracing packets",id:"tracing-packets",level:2},{value:"Internal network layout",id:"internal-network-layout",level:3},{value:"Capturing traffic inside the cluster",id:"capturing-traffic-inside-the-cluster",level:3},{value:"Traffic capture inside VPP",id:"traffic-capture-inside-vpp",level:3}
1,{value:"Traffic from a pod",id:"traffic-from-a-pod",level:4},{value:"Traffic from the phy",id:"traffic-from-the-phy",level:4},{value:"With Wireshark",id:"with-wireshark",level:4},{value:"Traffic received in the pods",id:"traffic-received-in-the-pods",level:3},{value:"Tcpdump is available on the host",id:"tcpdump-is-available-on-the-host",level:4},{value:"No tcpdump, but we have python !",id:"no-tcpdump-but-we-have-python-",level:4},{value:"Traffic to the kubelet agent",id:"traffic-to-the-kubelet-agent",level:3},{value:"Crashes & coredumps",id:"crashes--coredumps",level:2}],c={toc:s};function u(e){let{components:t,...o}=e;return(0,i.kt)("wrapper",(0,a.Z)({},c,o,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"vpp-dataplane-troubleshooting"},"VPP dataplane troubleshooting"),(0,i.kt)("h2",{id:"big-picture"},"Big picture"),(0,i.kt)("p",null,"This page describes the troubleshooting steps for the ",(0,i.kt)("a",{parentName:"p",href:"/calico/3.25/getting-started/kubernetes/vpp/getting-started"},"VPP dataplane"),". If you did not configure the VPP dataplane, this page is not for you!"),(0,i.kt)("p",null,"If you're encountering issues with the VPP dataplane, feel free to reach out to us either on the ",(0,i.kt)("a",{parentName:"p",href:"https://calicousers.slack.com/archives/C017220EXU1"},"#vpp channel")," on the Calico slack, or by opening a new issue in ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/projectcalico/vpp-dataplane/issues"},"GitHub"),")."),(0,i.kt)("h2",{id:"installing-calivppctl"},"Installing calivppctl"),(0,i.kt)("p",null,(0,i.kt)("inlineCode",{parentName:"p"},"calivppctl")," is a helper bash script shipped alongside vpp container images. It can be installed to your host with the following methods, and helps collecting logs and debugging a running cluster with the VPP dataplane installed."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"With curl")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"curl https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v3.25.1/test/scripts/vppdev.sh \\\n | tee /usr/bin/calivppctl\nchmod +x /usr/bin/calivppctl\n")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"With docker (and a cluster with calico-vpp running)")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"vppcontainer=$(docker ps | grep vpp_calico-vpp | awk '{ print $1 }')\ndocker cp ${vppcontainer}:/usr/bin/calivppctl /usr/bin/calivppctl\n")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"With kubectl (and a cluster with calico-vpp running)")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"vpppod=$(kubectl -n calico-vpp-dataplane get pods -o wide | grep calico-vpp-node- | awk '{ print $1 }' | head -1)\nkubectl -n calico-vpp-dataplane exec -it ${vpppod} -c vpp -- cat /usr/bin/calivppctl | tee /usr/bin/calivppctl > /dev/null\nchmod +x /usr/bin/calivppctl\n")),(0,i.kt)("h2",{id:"troubleshooting"},"Troubleshooting"),(0,i.kt)("h3",{id:"kubernetes-cluster"},"Kubernetes Cluster"),(0,i.kt)("p",null,"First you need to make sure Kubernetes is up and running."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"service kubelet status")," should give you a first hint."),(0,i.kt)("li",{parentName:"ul"},"Issues should be reported in the kubelet logs, which you can check with this command if you are using systemd: ",(0,i.kt)("inlineCode",{parentName:"li"},"journalctl -u kubelet -r -n200"))),(0,i.kt)("admonition",{type:"note"},(0,i.kt)("p",{parentName:"admonition"},"Kubernetes does not run with swap enabled.")),(0,i.kt)("h3",{id:"starting-calico-vpp-node-daemon-set"},"Starting calico-vpp-node Daemon set"),(0,i.kt)("p",null,"Once the cluster is correctly started, the next issue can come from the Daemonset configuration.\nBest is to start by inspecting the pods : are they running correctly ?\nUsually configuration issues (available hugepages, memory, ...) will be reported here"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"kubectl -n calico-vpp-dataplane describe pod/calico-vpp-node-XXXXX\n")),(0,i.kt)("admonition",{type:"note"},(0,i.kt)("p",{parentName:"admonition"},"If at this point you don't have enough hugepages, you'll have to restart kubelet\nafter allocating them for taking it into account (using for instance ",(0,i.kt)("inlineCode",{parentName:"p"},"service kubelet restart"),")")),(0,i.kt)("h3",{id:"having-vpp-up-and-running"},"Having VPP up and running"),(0,i.kt)("p",null,"Once the pods don't report any issue, the pods should have started. There are two\ncontainers for each node : VPP that starts the vpp process and setups connectivity,\nand the agent handling pod connectivity, service load balancing, BGP, policies, etc."),(0,i.kt)("p",null,"First check that VPP is running correctly. If the connectivity configuration, interface naming\nis not correct, this will be reported here. Once this is running, you should be able to ping your other nodes through VPP."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"}," Print VPP's log : basic connectivity and NIC configuration\ncalivppctl log -vpp myk8node1\n")),(0,i.kt)("p",null,"Then you can check for any issues reported by the Agent (e.g. BGP listen issue\nif the port is already taken, or missing configuration pieces). If this doesn't\nshow any errors, you should be able to ",(0,i.kt)("inlineCode",{parentName:"p"},"nslookup kubernetes.default")," from pods."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"}," Print the logs for the Calico VPP dataplane agent, programming serviceIPs, BGP, ...\ncalivppctl log -agent myk8node1\n")),(0,i.kt)("p",null,"If all this doesn't play well you can always use the export to generate an export.tar.gz\nbundle and ask for help on the ",(0,i.kt)("a",{parentName:"p",href:"https://calicousers.slack.com/archives/C017220EXU1"},"#vpp channel")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"calivppctl export\n")),(0,i.kt)("h2",{id:"accessing-the-vpp-cli"},"Accessing the VPP cli"),(0,i.kt)("p",null,"For further debugging, tracing packets and inspecting VPP's internals, you can\nget a vpp shell using the following"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"calivppctl vppctl myk8node1\n")),(0,i.kt)("h3",{id:"listing-interfaces-and-basics"},"Listing interfaces and basics"),(0,i.kt)("p",null,"To list existing interfaces and basic counters use"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp show int\nvpp show int addr\n")),(0,i.kt)("p",null,"To get more insights on the main interface (e.g. if you're using dpdk) you can check\nfor errors & drops in"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp show hardware-interfaces\n")),(0,i.kt)("p",null,"Other places to look for errors"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp show log # VPP startup log\nvpp show err # Prints out packet counters (not always actual errors, but includes drops)\nvpp show buffers # You should have non zero free buffers, otherwise traffic won't flow\n")),(0,i.kt)("h2",{id:"tracing-packets"}
1,"Tracing packets"),(0,i.kt)("h3",{id:"internal-network-layout"},"Internal network layout"),(0,i.kt)("p",null,"For starters, here is a small schematic of how the network looks like:\n",(0,i.kt)("img",{alt:"k8-calico-vpp",src:n(22297).Z,width:"581",height:"332"})),(0,i.kt)("p",null,"Container interfaces are named ",(0,i.kt)("inlineCode",{parentName:"p"},"tun[0-9]+"),". You can find which one belong to which container as follows."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},' Connect to vppctl\n$ calivppctl vppctl NODENAME\n\n List interfaces\nvpp show interface\n Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count\navf-0/d8/a/0 1 up 9000/0/0/0 tx packets 2\n tx bytes 216\nlocal0 0 down 0/0/0/0\ntap0 2 up 0/0/0/0 rx packets 9\n[...]\ntun3 5 up 0/0/0/0 rx packets 5\n rx bytes 431\n tx packets 5\n tx bytes 387\n ip4 5\n\n Show the route for address 11.0.166.132\nvpp show ip fib 11.0.166.132\nipv4-VRF:0, fib_index:0, flow hash:[src dst sport dport symmetric ] epoch:0 flags:none locks:[adjacency:1, default-route:1, ]\n11.0.166.132/32 fib:0 index:19 locks:5\n cnat refs:1 entry-flags:uRPF-exempt,interpose, src-flags:added,contributing,active, cover:-1 interpose:\n [@0]: [4] cnat-client:[11.0.166.132] tr:0 sess:1\n path-list:[26] locks:3 flags:shared, uPRF-list:24 len:1 itfs:[5, ]\n path:[32] pl-index:26 ip4 weight=1 pref=0 attached-nexthop: oper-flags:resolved, cfg-flags:attached,\n 11.0.166.132 tun3 (p2p)\n [@0]: ipv4 via 0.0.0.0 tun3: mtu:9000 next:7\n [...]\n\n This one is behind `tun3`\n If you want more info about this interface (name in Linux, queues, descriptors, ...)\nvpp show tun tun3\nInterface: tun3 (ifindex 5)\n name "eth0"\n host-ns "/proc/17675/ns/net"\n [...]\n')),(0,i.kt)("p",null,(0,i.kt)("inlineCode",{parentName:"p"},"tap0")," is the interface providing connectivity to the host, using the original interface name on the Linux side (use ",(0,i.kt)("inlineCode",{parentName:"p"},"show tap tap0")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"show ip punt redirect"),")."),(0,i.kt)("h3",{id:"capturing-traffic-inside-the-cluster"},"Capturing traffic inside the cluster"),(0,i.kt)("p",null,"Let's take the case of two pods talking to each other in your cluster (see the schema above).\nYou might want to inspect the traffic at 3 different locations :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"as it exits the pod (in Linux inside the first pod)"),(0,i.kt)("li",{parentName:"ul"},"as it goes through VPP"),(0,i.kt)("li",{parentName:"ul"},"as it is received in the second pod (in Linux again)")),(0,i.kt)("p",null,"We cover the three cases, first inside VPP (depending on where your traffic is coming from : a pod or outside your host)\nthen inside your pods (usually with tcpdump)"),(0,i.kt)("h3",{id:"traffic-capture-inside-vpp"},"Traffic capture inside VPP"),(0,i.kt)("h4",{id:"traffic-from-a-pod"},"Traffic from a pod"),(0,i.kt)("p",null,"The following snippet will allow you to capture all traffic coming from containers on a particular node, grep from a specific packet,\nand see what happened to it."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"}," Make sure that the trace buffer is clean in VPP\ncalivppctl vppctl NODENAME clear trace\n Add a trace from the virtio-input input-node\ncalivppctl vppctl NODENAME trace add virtio-input 500\n generate some traffic\ncalivppctl vppctl NODENAME show trace max 500 > somefile\n Grep for your IPs\ncat somefile | grep '1.2.3.4 -> 5.6.7.8' -A40 -B40\n")),(0,i.kt)("p",null,"Output looks quite cumbersome at first as it contains the whole path of a packet through VPP, from reception to tx."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp show trace\nPacket 1\n\n00:09:46:518858: virtio-input\n This packet has been received on the interface number #2 (column Idx in `show int`)\n and is 688 Bytes long\n virtio: hw_if_index 2 next-index 1 vring 0 len 688\n hdr: flags 0x00 gso_type 0x00 hdr_len 0 gso_size 0 csum_start 0 csum_offset 0 num_buffers 1\n00:09:46:518866: ip4-input\n we read TCP header, addresses and ports\n TCP: 20.0.0.1 -> 11.0.166.133\n tos 0x00, ttl 64, length 688, checksum 0x1bc5 dscp CS0 ecn NON_ECN\n fragment id 0x56fd, flags DONT_FRAGMENT\n TCP: 6443 -> 34112\n seq. 0xa1f93599 ack 0x818eb1c1\n flags 0x18 PSH ACK, tcp header: 32 bytes\n window 502, checksum 0x00b7\n00:09:46:518870: ip4-lookup\n fib 0 dpo-idx 5 flow hash: 0x00000000\n TCP: 20.0.0.1 -> 11.0.166.133\n tos 0x00, ttl 64, length 688, checksum 0x1bc5 dscp CS0 ecn NON_ECN\n fragment id 0x56fd, flags DONT_FRAGMENT\n TCP: 6443 -> 34112\n seq. 0xa1f93599 ack 0x818eb1c1\n flags 0x18 PSH ACK, tcp header: 32 bytes\n window 502, checksum 0x00b7\n00:09:46:518873: ip4-cnat-tx\n We need to do some NATing as it's Kubernetes\n found: session:[20.0.0.1;6443 -> 11.0.166.133;34112, TCP] => 11.96.0.1;443 -> 11.0.166.133;34112 lb:-1 age:4190\n00:09:46:518879: ip4-rewrite\n We rewrite the ip packet\n mac addresses only when coming / going to a PHY, as tun interfaces are L3-only\n tx_sw_if_index 6 dpo-idx 7 : ipv4 via 0.0.0.0 tun4: mtu:9000 next:8 flow hash: 0x00000000\n 00000000: 450002b056fd40003f0625650b6000010b00a68501bb8540a1f93599818eb1c1\n 00000020: 801801f620c700000101080a3f906c98fbaaba031703030277413d39\n Output happens on the interface `tun4`\n00:09:46:518880: tun4-output\n tun4\n 00000000: 450002b056fd40003f0625650b6000010b00a68501bb8540a1f93599818eb1c1\n 00000020: 801801f620c700000101080a3f906c98fbaaba031703030277413d39b97817c1\n 00000040: 41392fdbe0e9d4886849851476cdb8986362ee2f789bfefd8a5c106c898d1309\n 00000060: 4f8f8cb89159d99e986813a48d91334930eb5eb10ca4248c\n00:09:46:518881: tun4-tx\n buffer 0x24cf615: current data 0, length 688, buffer-pool 1, ref-count 1, totlen-nifb 0, trace handle 0x1000000\n ipv4 tcp hdr-sz 52 l2-hdr-offset 0 l3-hdr-offset 0 l4-hdr-offset 20 l4-hdr-sz 32\n 0x0b60: 40:00:3f:06:25:65 -> 45:00:02:b0:56:fd\n\nPacket 2\n[...]\n")),(0,i.kt)("h4",{id:"traffic-from-the-phy"},"Traffic from the phy"),(0,i.kt)("p",null,"If you want to capture traffic coming from the physical NIC, you should use ",(0,i.kt)("inlineCode",{parentName:"p"},"trace add")," but with a different source node a.k.a ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk-input")," ",(0,i.kt)("inlineCode",{parentName:"p"},"af-packet-input")," ",(0,i.kt)("inlineCode",{parentName:"p"},"af_xdp-input")," ",(0,i.kt)("inlineCode",{parentName:"p"},"avf-input")," instead of ",(0,i.kt)("inlineCode",{parentName:"p"},"virtio-input"),"."),(0,i.kt)("p",null,(0,i.kt)("inlineCode",{parentName:"p"},"show run")," should give you a hint of the ",(0,i.kt)("inlineCode",{parentName:"p"},"X-input")," node you want to trace from."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp show run\nThread 1 vpp_wk_0 (lcore 25)\nTime 1.9, 10 sec internal node vector rate 1.05 loops/sec 1074819.68\n vector rates in 7.5356e0, out 7.5356e0, drop 0.0000e0, punt 0.0000e0\n Name State Calls Vectors Suspends Clocks Vectors/Call\navf-input polling 2233530 0 0 8.24e1 0.00\nip4-cnat-snat active 1 1 0 5.35e3 1.00\nip4-cnat-tx active 14 15 0 1.18e3 1.07\n[...]\n\n Here we seem to want to use trace add avf-input 200\n")),(0,i.kt)("p",null,"Same as with traffic from a container, you can use"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"}," Make sure that the trace buffer is clean in VPP\ncalivppctl vppctl NODENAME clear trace\n Add a trace from the virtio-input input-node\ncalivppctl vppctl NODENAME trace add avf-input 500\n generate some traffic\ncalivppctl vppctl NODENAME show trace max 500 > somefile\n Grep for your IPs\ncat somefile | grep '1.2.3.4 -> 5.6.7.8' -A40 -B40\n")),(0,i.kt)("h4",{id:"with-wireshark"},"With Wireshark"),(0,i.kt)("p",null,"Alternatively to the trace, you can do a capture and analyze it inside Wireshark. You can do this with:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"vpp pcap dispatch trace on max 1000 file vppcapture buffer-trace dpdk-input 1000\nvpp pcap dispatch trace off\n")),(0,i.kt)("p",null,"This will generate a file named ",(0,i.kt)("inlineCode",{parentName:"p"},"/tmp/vppcapture")),(0,i.kt)("p",null,"Then on your host run:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"calivppctl sh vpp NODENAME\nroot@server:~ mv /tmp/vppcapture /var/lib/vpp/\nroot@server:~ exit\n The file should now be at /var/lib/vpp/vppcapture on your host 'NODENAME'\n")),(0,i.kt)("p",null,"You can then ",(0,i.kt)("inlineCode",{parentName:"p"},"scp NODENAME:/var/lib/vpp/vppcapture .")," on your machine and open it with Wireshark\n",(0,i.kt)("a",{parentName:"p",href:"https://haryachyy.wordpress.com/2019/09/29/learning-vpp-trace-with-wireshark/"},"More info about this here")),(0,i.kt)("h3",{id:"traffic-received-in-the-pods"},"Traffic received in the pods"),(0,i.kt)("p",null,"To inspect traffic actually received by the pods (if ",(0,i.kt)("inlineCode",{parentName:"p"},"tcpdump")," is installed in the pod), simply run ",(0,i.kt)("inlineCode",{parentName:"p"},"tcpdump -ni eth0")," inside the pod. If tcpdump is not available in the pod, here are two options to still be able to capture pod traffic:"),(0,i.kt)("h4",{id:"tcpdump-is-available-on-the-host"},"Tcpdump is available on the host"),(0,i.kt)("p",null,"Provided that you have ",(0,i.kt)("inlineCode",{parentName:"p"},"tcpdump")," installed on the host, you can use ",(0,i.kt)("inlineCode",{parentName:"p"},"nsenter")," to attach to the pod's network namespace and use the host's ",(0,i.kt)("inlineCode",{parentName:"p"},"tcpdump")," on the container's interface."),(0,i.kt)("p",null,"This works on docker as follows :"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"\n Find the container ID you want to inspect\n$ docker ps\nCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n4c01db0b339c ubuntu:12.04 bash 17 seconds ago Up 16 seconds 3300-3310/tcp webapp\n\n Get the container PID out of it\n$ docker inspect --format '{{ .State.Pid }}' 4c01db0b339c\n12345\n\n Attach\n$ nsenter -t 12345 -n bash\n\n")),(0,i.kt)("h4",{id:"no-tcpdump-but
1-we-have-python-"},"No tcpdump, but we have python !"),(0,i.kt)("p",null,"Open an AF_PACKET socket in python with the following code\nand run it attached to the running namespace as previously."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'#!/usr/bin/env python\nfrom socket import *\nfrom struct import unpack\n\nIFNAME = "eth0"\nN_PKT = 50\nMTU=1500\n\nsock = socket(AF_PACKET, SOCK_DGRAM, 0x0800)\nsock.bind((IFNAME, 0x0800))\nfor _ in range(N_PKT):\n data = sock.recvfrom(MTU, 0)[0]\n src_addr = inet_ntop(AF_INET, data[12:16])\n dst_addr = inet_ntop(AF_INET, data[16:20])\n src_port, = unpack("!H", data[20:22])\n dst_port, = unpack("!H", data[22:24])\n data_len, = unpack("!H", data[24:26])\n cksum, = unpack("!H", data[26:28])\n\n print("%s:%d -> %s:%d len %d cs %d" % (src_addr, src_port, dst_addr, dst_port, data_len, cksum))\n')),(0,i.kt)("p",null,"This requires privileges and thus is usually easier to run from the host. From the host, you can use ",(0,i.kt)("inlineCode",{parentName:"p"},'echo "the python blob above" | nsenter -t <thePID> -n python')," to execute this code."),(0,i.kt)("h3",{id:"traffic-to-the-kubelet-agent"},"Traffic to the kubelet agent"),(0,i.kt)("p",null,"As the kubelet agent runs directly on the host without a network namespace, pods talking to it (e.g. coredns resolvers) would go through a specific path. Packets destined to it will be caught by VPP's punt mechanism, and will be forwarded to the host through a tap interface which will have the same name as the original interface in Linux."),(0,i.kt)("p",null,"To debug traffic within VPP, use the trace & check that traffic is correctly punted to the tap0 interface."),(0,i.kt)("p",null,"On the host, you can use ",(0,i.kt)("inlineCode",{parentName:"p"},"tcpdump")," normally to check the traffic."),(0,i.kt)("h2",{id:"crashes--coredumps"},"Crashes & coredumps"),(0,i.kt)("p",null,"To instruct vpp to leave a coredump in the event of a crash, you can pass the ",(0,i.kt)("inlineCode",{parentName:"p"},"CALICOVPP_CORE_PATTERN")," environment variable to the vpp container:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: calico-vpp-node\n\n---\n- name: vpp\n env:\n - name: CALICOVPP_CORE_PATTERN\n value: '/home/hostuser/vppcore.%e.%p'\n volumeMounts:\n - name: userhome\n mountPath: /home/hostuser\n\n---\nvolumes:\n - name: userhome\n hostPath:\n path: ${SOME_DIRECTORY}\n")),(0,i.kt)("p",null,"This will generate a ",(0,i.kt)("inlineCode",{parentName:"p"},"vppcore.vpp_main.<pid>")," file in ",(0,i.kt)("inlineCode",{parentName:"p"},"${DIR}")," if vpp aborts unexpectedly. If you encounter this situation, please note the exact version of the vpp image that generated the corefile (using the image hash) to facilitate further troubleshooting."),(0,i.kt)("p",null,"To explore it run:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-bash"},"docker run -it --entrypoint=bash -v $DIR/vppcore.vpp_main.12345:/root/vppcore calicovpp/vpp:VERSION\n You should have a shell inside the vpp container\napt update && apt install -y gdb\ngdb vpp ./vppcore\n")))}u.isMDXComponent=!0},22297:(e,t,n)=>{n.d(t,{Z:()=>a});const a=n.p+"assets/images/vpp-tracing-net-ae0b996dcce1b8f861f46a201518a3dd.svg"}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.