1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[5858],{3905:(e,t,n)=>{n.d(t,{Zo:()=>s,kt:()=>m});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var p=a.createContext({}),d=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},s=function(e){var t=d(e.components);return a.createElement(p.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},h=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,p=e.parentName,s=o(e,["components","mdxType","originalType","parentName"]),h=d(n),m=i,u=h["".concat(p,".").concat(m)]||h[m]||c[m]||r;return n?a.createElement(u,l(l({ref:t},s),{},{components:n})):a.createElement(u,l({ref:t},s))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,l=new Array(r);l[0]=h;var o={};for(var p in t)hasOwnProperty.call(t,p)&&(o[p]=t[p]);o.originalType=e,o.mdxType="string"==typeof e?e:i,l[1]=o;for(var d=2;d<r;d++)l[d]=n[d];return a.createElement.apply(null,l)}return a.createElement.apply(null,n)}h.displayName="MDXCreateElement"},5602:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>l,default:()=>c,frontMatter:()=>r,metadata:()=>o,toc:()=>d});var a=n(7462),i=(n(7294),n(3905));const r={},l="VPP Implementation",o={unversionedId:"developer/VPP_Implementation",id:"developer/VPP_Implementation",title:"VPP Implementation",description:"This document describes Terragraph's datapath implementation using VPP.",source:"@site/../docs/developer/VPP_Implementation.md",sourceDirName:"developer",slug:"/developer/VPP_Implementation",permalink:"/docs/developer/VPP_Implementation",draft:!1,editUrl:"https://github.com/terragraph/meta-terragraph/edit/main/docs/../docs/developer/VPP_Implementation.md",tags:[],version:"current",frontMatter:{},sidebar:"developerManualSidebar",previous:{title:"Driver Stack",permalink:"/docs/developer/Driver_Stack"},next:{title:"Timing and Synchronization",permalink:"/docs/developer/Timing_Synchronization"}},p={},d=[{value:"Overview",id:"overview",level:2},{value:"Versioning",id:"versioning",level:3},{value:"Code Structure",id:"code-structure",level:3},{value:"VPP Services",id:"vpp-services",level:3},{value:"VPP Packet Processing Graph",id:"vpp-packet-processing-graph",level:3}
1,{value:"Stock VPP Datapath Overview",id:"stock-vpp-datapath-overview",level:2},{value:"Terragraph Config Plugin (vpp-tgcfg)",id:"terragraph-config-plugin-vpp-tgcfg",level:2},{value:"Interfaces",id:"interfaces",level:3},{value:"Configuration",id:"configuration",level:3},{value:"Graph Nodes",id:"graph-nodes",level:3},{value:"tg-link-input",id:"tg-link-input",level:4},{value:"tg-slowpath-wired-rx, tg-slowpath-terra-rx",id:"tg-slowpath-wired-rx-tg-slowpath-terra-rx",level:4},{value:"tg-link-local-tx",id:"tg-link-local-tx",level:4},{value:"tg-wired-local-tx",id:"tg-wired-local-tx",level:4},{value:"vpp-terra-tx",id:"vpp-terra-tx",level:4},{value:"tg-link-local-rx",id:"tg-link-local-rx",level:4},{value:"tg-wired-local-rx",id:"tg-wired-local-rx",level:4},{value:"HQoS Scheduler (TGHQoS)",id:"hqos-scheduler-tghqos",level:2},{value:"Code Structure",id:"code-structure-1",level:3},{value:"Hierarchy",id:"hierarchy",level:3},{value:"RED Dropper",id:"red-dropper",level:3},{value:"CLI Commands",id:"cli-commands",level:3},{value:"HQoS Operation",id:"hqos-operation",level:3},{value:"Wigig Jumbo Frame Support",id:"wigig-jumbo-frame-support",level:2},{value:"VPP and DPDK Buffers",id:"vpp-and-dpdk-buffers",level:3},{value:"Jumbo Frame Segmentation",id:"jumbo-frame-segmentation",level:3},{value:"Transmitting / Segmentation",id:"transmitting--segmentation",level:4},{value:"Receiving / Reassembly",id:"receiving--reassembly",level:4},{value:"VPP Threads & Core Allocation",id:"vpp-threads--core-allocation",level:2},{value:"Interface Thread Assignment",id:"interface-thread-assignment",level:3},{value:"Firmware Message Paths",id:"firmware-message-paths",level:2},{value:"Firmware <code>ioctl</code>",id:"firmware-ioctl",level:3},{value:"Firmware Event",id:"firmware-event",level:3},{value:"Debugging",id:"debugging",level:2},{value:"Use Debug Builds",id:"use-debug-builds",level:3},{value:"Enable Core Dumps",id:"enable-core-dumps",level:3},{value:"Collect Packet Traces",id:"collect-packet-traces",level:3},{value:"Resources",id:"resources",level:2}],s={toc:d};function c(e){let{components:t,...n}=e;return(0,i.kt)("wrapper",(0,a.Z)({},s,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"vpp-implementation"},"VPP Implementation"),(0,i.kt)("p",null,"This document describes Terragraph's datapath implementation using VPP."),(0,i.kt)("h2",{id:"overview"},"Overview"),(0,i.kt)("p",null,"Terragraph uses ",(0,i.kt)("a",{parentName:"p",href:"https://wiki.fd.io/view/VPP"},"VPP")," (Vector Packet Processing framework), along with ",(0,i.kt)("a",{parentName:"p",href:"https://www.dpdk.org/"},"DPDK"),"\nand a custom wil6210 PMD, to implement its datapath."),(0,i.kt)("h3",{id:"versioning"},"Versioning"),(0,i.kt)("p",null,"Terragraph uses the NXP fork of standard VPP, with a large number of\nTerragraph-specific patches applied on top. The current version is NXP release\n21.08-LSDK, corresponding to DPDK 20.11.2 and VPP 21.01."),(0,i.kt)("h3",{id:"code-structure"},"Code Structure"),(0,i.kt)("p",null,"VPP is installed via ",(0,i.kt)("inlineCode",{parentName:"p"},"meta-qoriq/recipes-extended/vpp/vpp_21.08-lsdk.bb"),". There\nare also some scripts related to VPP service management and startup\nconfiguration installed via ",(0,i.kt)("inlineCode",{parentName:"p"},"meta-qoriq/recipes-facebook/tg-vpp/tg-vpp_0.1.bb"),"."),(0,i.kt)("p",null,"Terragraph builds additional features on top of VPP, summarized below:"),(0,i.kt)("table",null,(0,i.kt)("thead",{parentName:"table"},(0,i.kt)("tr",{parentName:"thead"},(0,i.kt)("th",{parentName:"tr",align:null},"Name"),(0,i.kt)("th",{parentName:"tr",align:null},"Type"),(0,i.kt)("th",{parentName:"tr",align:null},"Source Location"),(0,i.kt)("th",{parentName:"tr",align:null},"Description"))),(0,i.kt)("tbody",{parentName:"table"},(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp-tgcfg")),(0,i.kt)("td",{parentName:"tr",align:null},"plugin"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"src/vpp_plugins/tgcfg/")),(0,i.kt)("td",{parentName:"tr",align:null},"Configures the Terragraph-specific slow datapath.")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp-chaperone")),(0,i.kt)("td",{parentName:"tr",align:null},"application"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"src/vpp_plugins/vpp-chaperone/")),(0,i.kt)("td",{parentName:"tr",align:null},"Applies user configs to VPP (e.g. POP, CPE, SR) over the shared memory API.")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"openr-fib-vpp")),(0,i.kt)("td",{parentName:"tr",align:null},"application"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"src/vpp_plugins/openr-fib-vpp/")),(0,i.kt)("td",{parentName:"tr",align:null},"The ",(0,i.kt)("a",{parentName:"td",href:"https://github.com/facebook/openr"},"Open/R")," FIB (forwarding information base) implementation for VPP.")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp-ptptc")),(0,i.kt)("td",{parentName:"tr",align:null},"plugin"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"src/vpp_plugins/ptptc/")),(0,i.kt)("td",{parentName:"tr",align:null}
1,"Configures PTP-TC (Precision Timing Protocol Transparent Clock) timestamping in VPP.")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp-esmc")),(0,i.kt)("td",{parentName:"tr",align:null},"plugin"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"src/vpp_plugins/esmc/")),(0,i.kt)("td",{parentName:"tr",align:null},"Configures SyncE (Synchronous Ethernet) ESMC (Ethernet Synchronization Messaging Channel) operation in VPP.")))),(0,i.kt)("h3",{id:"vpp-services"},"VPP Services"),(0,i.kt)("p",null,"The following VPP services are managed by runit: ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp"),", ",(0,i.kt)("inlineCode",{parentName:"p"},"fib_vpp"),", and\n",(0,i.kt)("inlineCode",{parentName:"p"},"vpp_chaperone"),". Logs for each service are written to\n",(0,i.kt)("inlineCode",{parentName:"p"},"/var/log/<service>/current")," by svlogd. Additionally, DPDK/PMD logs captured by\nVPP are extracted from syslog and written to ",(0,i.kt)("inlineCode",{parentName:"p"},"/var/log/vpp/vnet.log"),". For more\ndetails, see ",(0,i.kt)("a",{parentName:"p",href:"/docs/developer/Service_Scripts#service-scripts-node-services"},"Node Services"),"."),(0,i.kt)("p",null,"The lifetimes of certain services are tied together:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Restarting ",(0,i.kt)("inlineCode",{parentName:"li"},"e2e_minion")," will also restart ",(0,i.kt)("inlineCode",{parentName:"li"},"vpp")," (since the entire driver stack\ngets unloaded)"),(0,i.kt)("li",{parentName:"ul"},"Restarting ",(0,i.kt)("inlineCode",{parentName:"li"},"vpp")," will also forcibly restart ",(0,i.kt)("inlineCode",{parentName:"li"},"fib_vpp")," (since it uses VPP's SHM\nAPI)"),(0,i.kt)("li",{parentName:"ul"},"Restarting ",(0,i.kt)("inlineCode",{parentName:"li"},"fib_vpp")," ",(0,i.kt)("em",{parentName:"li"},"may")," re-run ",(0,i.kt)("inlineCode",{parentName:"li"},"vpp_chaperone")," (on POP nodes)")),(0,i.kt)("p",null,"Startup configuration for VPP is generated at boot time by\n",(0,i.kt)("inlineCode",{parentName:"p"},"src/terragraph-e2e/lua/update_vpp_startup_conf.lua")," using parameters read from\nthe node configuration and node info files."),(0,i.kt)("p",null,"A separate ",(0,i.kt)("inlineCode",{parentName:"p"},"coop")," service monitors the Linux loopback prefix for changes and\nreconfigures the VPP loopback prefix if necessary by running ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp_chaperone"),"."),(0,i.kt)("p",null,"Some VPP stats are scraped by the ",(0,i.kt)("inlineCode",{parentName:"p"},"stats_agent")," process and published on a\nZMQ port (see ",(0,i.kt)("a",{parentName:"p",href:"/docs/developer/Stats_Events_Logs"},"Stats, Events, Logs")," for details)."),(0,i.kt)("h3",{id:"vpp-packet-processing-graph"},"VPP Packet Processing Graph"),(0,i.kt)("p",null,"The flowchart below demonstrates the path that data packets take through VPP and\nspecial hooks established in the VPP packet processing graph by the ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-tgcfg"),"\nplugin. This is explained in detail in subsequent sections."),(0,i.kt)("p",{align:"center"},(0,i.kt)("img",{src:"/figures/tgcfg.svg",width:"1000"})),(0,i.kt)("h2",{id:"stock-vpp-datapath-overview"}
1,"Stock VPP Datapath Overview"),(0,i.kt)("p",null,"All data packets received from hardware enter VPP through the ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk-input")," node.\nThis node normally passes data to the ",(0,i.kt)("inlineCode",{parentName:"p"},"ethernet-input")," node, which then\nclassifies packets as IPv4, IPv6, WPA, and so on. This document will focus on\nIPv6 packets, which are the primary data packet type being dealt with."),(0,i.kt)("p",null,"Each IPv6 packet ends up in the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-input"),' node, which is mostly responsible\nfor determining if the packet is supposed to transit out of the node. This\ninvolves somewhat complex logic that is covered by the "Local?" decision node,\nwithout going into details on how this is actually implemented. For example, all\nunicast packets get their destination address looked up in the IPv6 FIB. All\nlink-local addresses will hit an entry in the FIB that will redirect them to the\n',(0,i.kt)("inlineCode",{parentName:"p"},"ip6-link-local")," node, which in turn will perform a lookup in a separate FIB to\nfind out if the destination address matches one of the interfaces that VPP\nrecognizes. If it does, the packet gets forwarded into the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," node to\nbe handled on the node. Otherwise, the packet is for an invalid address and gets\ndropped."),(0,i.kt)("p",null,"The FIB entry to redirect all link-local packets to ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-link-local")," looks like\nthis:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"fe80::/10\nunicast-ip6-chain\n[@0]: dpo-load-balance: [proto:ip6 index:7 buckets:1 uRPF:6 to:[162:22458]]\n[0] [@15]: ip6-link-local\n")),(0,i.kt)("p",null,"The following trace fragment demonstrates the handling of link-local packets in\nVPP by this path:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"00:53:14:979091: ip6-lookup\n fib 0 dpo-idx 0 flow hash: 0x00000000\n ICMP6: fe80::6ce:14ff:feff:4281 -> fe80::6ce:14ff:feff:428b\n tos 0x00, flow label 0xb271e, hop limit 64, payload length 64\n ICMP echo_request checksum 0xfd87\n00:53:14:979102: ip6-link-local\n sw_if_index:14 fib_index:10\n00:53:14:979106: ip6-lookup\n fib 10 dpo-idx 14 flow hash: 0x00000000\n ICMP6: fe80::6ce:14ff:feff:4281 -> fe80::6ce:14ff:feff:428b\n tos 0x00, flow label 0xb271e, hop limit 64, payload length 64\n ICMP echo_request checksum 0xfd87\n00:53:14:979110: ip6-local\n ICMP6: fe80::6ce:14ff:feff:4281 -> fe80::6ce:14ff:feff:428b\n tos 0x00, flow label 0xb271e, hop limit 64, payload length 64\n ICMP echo_request checksum 0xfd87\n")),(0,i.kt)("p",null,"Similarly, all multicast addresses are looked up in the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-mfib")," FIB table and\neither get dropped or get forwarded to ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," to be handled on the node:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"00:52:58:405106: ethernet-input\n IP6: 04:ce:14:ff:42:81 -> 04:ce:14:ff:42:8b\n00:52:58:405118: ip6-input\n UDP: fe80::6ce:14ff:feff:4281 -> ff02::1\n tos 0xc0, flow label 0x5d69e, hop limit 255, payload length 171\n UDP: 6666 -> 6666\n length 171, checksum 0x63e7\n00:52:58:405128: ip6-mfib-forward-lookup\n fib 0 entry 4\n00:52:58:405136: ip6-mfib-forward-rpf\n entry 4 itf 14 flags Accept,\n00:52:58:405144: ip6-replicate\n replicate: 2 via [@1]: dpo-receive\n00:52:58:405153: ip6-local\n UDP: fe80::6ce:14ff:feff:4281 -> ff02::1\n tos 0xc0, flow label 0x5d69e, hop limit 255, payload length 171\n UDP: 6666 -> 6666\n length 171, checksum 0x63e7\n")),(0,i.kt)("p",null,"All known unicast addresses for existing interfaces in VPP will get an entry in\nthe IPv6 FIB, similar to this one, which will redirect matching packets to the\n",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," node as well:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"71::3/128\n unicast-ip6-chain\n [@0]: dpo-load-balance: [proto:ip6 index:66 buckets:1 uRPF:71 to:[3:312]]\n [0] [@2]: dpo-receive: 71::3 on loop1\n")),(0,i.kt)("p",null,"A sample trace fragment is given below:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"01:21:15:353214: ethernet-input\n IP6: 04:ce:14:ff:42:81 -> 04:ce:14:ff:42:8b\n01:21:15:353224: ip6-input\n ICMP6: 2801:b8:fb:fb9c::1 -> 71::3\n tos 0x00, flow label 0x27108, hop limit 63, payload length 64\n ICMP echo_request checksum 0xbd02\n01:21:15:353233: ip6-lookup\n fib 0 dpo-idx 5 flow hash: 0x00000000\n ICMP6: 2801:b8:fb:fb9c::1 -> 71::3\n tos 0x00, flow label 0x27108, hop limit 63, payload length 64\n ICMP echo_request checksum 0xbd02\n01:21:15:353243: ip6-local\n ICMP6: 2801:b8:fb:fb9c::1 -> 71::3\n tos 0x00, flow label 0x27108, hop limit 63, payload length 64\n ICMP echo_request checksum 0xbd02\n01:21:15:353252: tg-slowpath-terra-rx\n")),(0,i.kt)("p",null,"The important takeaway from the above examples is as follows: any packet that is\nto be handled on the node itself eventually goe
1s though a FIB lookup that\nreturns a ",(0,i.kt)("inlineCode",{parentName:"p"},"dpo-receive")," object. This tells VPP that the packet should be handled\nby the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local"),' node as the next step. The DPO ("data-path object") also\nencodes the interface for which the packet is received, so upon entry\n',(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," knows the interface that has received the packet and the interface\nfor which the packet was sent."),(0,i.kt)("p",null,"All other packets either match a valid entry in the FIB and are forwarded to the\nappropriate interface to be sent out, or they are dropped as invalid."),(0,i.kt)("h2",{id:"terragraph-config-plugin-vpp-tgcfg"},"Terragraph Config Plugin (vpp-tgcfg)"),(0,i.kt)("p",null,"The ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-tgcfg")," plugin installs several hooks into the VPP packet graph and\nconfigures certain interfaces and their properties on startup. The primary goal\nfor these is to handle the Terragraph-specific slow datapath, where some traffic\nis handled entirely in VPP (fast path) and some gets forwarded to the Linux\nkernel to be handled there (slow path)."),(0,i.kt)("p",null,'The slow path is enabled by default. It can be switched off by adding the\n"slowpath" directive in the "terragraph" section of VPP\'s startup\nconfiguration:'),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"terragraph {\n slowpath off\n}\n")),(0,i.kt)("h3",{id:"interfaces"},"Interfaces"),(0,i.kt)("p",null,'VPP maintains several "split-level" interfaces:'),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"vpp-terraX")," interfaces, representing Terragraph links. Each instance of a\n",(0,i.kt)("inlineCode",{parentName:"li"},"vpp-terraX")," interface has a corresponding ",(0,i.kt)("inlineCode",{parentName:"li"},"terraX")," interface on the Linux\nside. The plugin uses a side-channel interface with the wil6210 PMD and\n",(0,i.kt)("inlineCode",{parentName:"li"},"dpdk-dhd.ko")," kernel module to receive packets from the kernel and to inject\npackets into the kernel on behalf of ",(0,i.kt)("inlineCode",{parentName:"li"},"terraX"),"."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"vpp-nicX")," tap interfaces, and corresponding ",(0,i.kt)("inlineCode",{parentName:"li"},"nicX")," endpoints on the Linux\nside. These represent wired Ethernet interfaces and use VPP's built-in\n",(0,i.kt)("inlineCode",{parentName:"li"},"tapcli")," module to read and inject packets."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"vpp-vnet0")," tap interfaces, and corresponding ",(0,i.kt)("inlineCode",{parentName:"li"},"vnet0")," endpoints on the Linux\nside. This pair is used for L3-routable connection between vpp and the Linux\nkernel. Namely, DNs and CNs will have a default route pointing to ",(0,i.kt)("inlineCode",{parentName:"li"},"vnet0"),"."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"loop0"),", a VPP interface that serves as a pure VPP-only L3-routable endpoint.\nIt gets assigned a ",(0,i.kt)("inlineCode",{parentName:"li"},"::2")," address from the node's prefix for that interface by\nanother process (",(0,i.kt)("inlineCode",{parentName:"li"},"vpp_chaperone"),"). The only useful function of this interface\nat the time of this writing is to serve its global IPv6 address to the ICMPv6\nerror source address selection algorithm.")),(0,i.kt)("h3",{id:"configuration"},"Configuration"),(0,i.kt)("p",null,"The VPP plugin identifies all Wigig interfaces at startup time, establishes a\nside-channel API connection to corresponding PMD and consequently ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk-dhd.ko"),'\nrunning in the kernel automatically. This auto-probe can be switched off by\nadding the "auto-probe off" directive in the "terragraph" section of VPP\'s\nstartup configuration:'),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"terragraph {\n auto-probe off\n}\n")),(0,i.kt)("p",null,"VPP creation of the ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-vnet0"),"/",(0,i.kt)("inlineCode",{parentName:"p"},"vnet0")," tap interface pair is enabled by the\nfollowing directive in the startup configuration:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"terragraph {\n host interface vnet0\n}\n")),(0,i.kt)("p",null,"The ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-nic0"),"/",(0,i.kt)("inlineCode",{parentName:"p"},"nic0")," tap interface pair is created by the following directive\nin the startup configuration:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"terragraph {\n interface TenGigabitEthernet0 {\n tap nic0\n }\n}\n")),(0,i.kt)("p",null,"The directive example above will create ",(0,i.kt)("inlineCode",{parentName:"p"},"nic0")," on the kernel side and ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-nic0"),"\nin VPP, and arrange for slow path processing similar to processing done for\nWigig/",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terra0"),"/",(0,i.kt)("inlineCode",{parentName:"p"},"terra0"),". The MAC address of the ",(0,i.kt)("inlineCode",{parentName:"p"},"nic0")," interface in the\nkernel will match the MAC address of the corresponding ",(0,i.kt)("inlineCode",{parentName:"p"},"TenGigabitEthernet0"),"\ninterface in VPP."),(0,i.kt)("h3",{id:"graph-nodes"},"Graph Nodes"),(0,i.kt)("p",null,"The following section lists the graph nodes that ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-tgcfg")," implements, their\nfunctions, and where they are connected into the packet processing graph."),(0,i.kt)("h4",{id:"tg-link-input"},"tg-link-input"),(0,i.kt)("p",null,"This node intercepts all packets received from ",(0,i.kt)("inlineCode",{parentName:"p"},"WigigX/Y/Z/0")," interfaces and\nexamines packet metadata to find out what ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," interface should be\nmarked as the receiving interface for the packet. The packet metadata is stored\nin the ",(0,i.kt)("inlineCode",{parentName:"p"},"rte_mbuf")," structure using a custom dynamic field (or ",(0,i.kt)("inlineCode",{parentName:"p"},"dynfield"),"). It\nthen forwards the packets to the normal VPP data processing path, namely the\n",(0,i.kt)("inlineCode",{parentName:"p"},"ethernet-input")," node. As far as VPP is concerned, the packets are received by\nproper ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," interfaces from now on."),(0,i.kt)("p",null,"The node is inserted into the graph by a call in the ",(0,i.kt)("inlineCode",{parentName:"p"},"tg_interface_enable()"),"\nfunction:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},' /* Let incoming traffic to be assigned to correct link interface */\n vnet_feature_enable_disable ("device-input", "tg-link-input", sw_if_index,\n enable_disable, 0, 0
1);\n')),(0,i.kt)("h4",{id:"tg-slowpath-wired-rx-tg-slowpath-terra-rx"},"tg-slowpath-wired-rx, tg-slowpath-terra-rx"),(0,i.kt)("p",null,"These two nodes intercept all packets from ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," and slow path-enabled\nwired ports hitting the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," node, before any processing has been done\nfor them. These nodes examine the interface addressed by the packet and if it\nhappens to belong to any slow path-enabled interface, the packet is forwarded to\n",(0,i.kt)("inlineCode",{parentName:"p"},"tg-link-local-tx")," or ",(0,i.kt)("inlineCode",{parentName:"p"},"tg-wired-local-tx")," nodes for packets received over Wigig\nor over wired ports, respectively. The packet will then be forwarded to the\nkernel and inserted into the Linux stack as appropriate."),(0,i.kt)("p",null,"Packets addressed to any other interface are processed by VPP locally. Packets\naddressing ",(0,i.kt)("inlineCode",{parentName:"p"},"loop0")," fall into this category."),(0,i.kt)("p",null,"Instantiated as features on the ",(0,i.kt)("inlineCode",{parentName:"p"},"ip6-local")," arc in the\n",(0,i.kt)("inlineCode",{parentName:"p"},"tg_wired_interface_enable()")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"tg_interface_enable()")," functions."),(0,i.kt)("h4",{id:"tg-link-local-tx"},"tg-link-local-tx"),(0,i.kt)("p",null,"This node accepts packets and calls into the PMD/dpdk-dhd to get them delivered\nto the kernel. The kernel module then injects packets onto the Linux networking\nstack using ",(0,i.kt)("inlineCode",{parentName:"p"},"netif_rx")," on behalf of the proper ",(0,i.kt)("inlineCode",{parentName:"p"},"terraX")," interface. The node is\nscheduled as the next node explicitly by ",(0,i.kt)("inlineCode",{parentName:"p"},"tg-slowpath-terra-rx")," when necessary."),(0,i.kt)("h4",{id:"tg-wired-local-tx"},"tg-wired-local-tx"),(0,i.kt)("p",null,"This node accepts packets that came from a slow path-enabled wired port (e.g.\n",(0,i.kt)("inlineCode",{parentName:"p"},"TenGigabitEthernet0"),"), and forwards them to the interface output of the\ncorresponding ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-nicX")," interface. This in turn will make ",(0,i.kt)("inlineCode",{parentName:"p"},"tapcli-tx")," send\npackets over to the Linux side and be inserted into the Linux kernel stack on\nbehalf of the matching ",(0,i.kt)("inlineCode",{parentName:"p"},"nicX")," interface. The node is scheduled as the next node\nexplicitly by ",(0,i.kt)("inlineCode",{parentName:"p"},"tg-slowpath-wired-rx")," when necessary."),(0,i.kt)("h4",{id:"vpp-terra-tx"},"vpp-terra-tx"),(0,i.kt)("p",null,"This node accepts all packets that VPP wants to be sent over the air using one\nof the ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," interfaces as the source. Since ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," are virtual\ninterfaces representing the link, the node performs a function logically\nopposite to that of the ",(0,i.kt)("inlineCode",{parentName:"p"},"tg-link-input")," node: it uses ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," information\nto mark the packet with the desired link to be used and then forwards the packet\nto ",(0,i.kt)("inlineCode",{parentName:"p"},"WigigX/Y/Z/0")," for actual transmission."),(0,i.kt)("p",null,"This is not an actual node, but rather a TX function ",(0,i.kt)("inlineCode",{parentName:"p"},"tg_link_interface_tx()"),"\nregistered as the output handler of the ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX")," interface class. VPP\ncreates ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-terraX-tx")," nodes with this function as the handler automatically."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},'VNET_DEVICE_CLASS (tg_link_interface_device_class) = {\n .name = "TGLink",\n .format_device_name = format_tg_link_name,\n .format_tx_trace = format_tg_link_tx_trace,\n .tx_function = tg_link_interface_tx,\n .admin_up_down_function = tg_link_interface_admin_up_down,\n};\n')),(0,i.kt)("h4",{id:"tg-link-local-rx"},"tg-link-local-rx"),(0,i.kt)("p",null,"This node gets scheduled every time the kernel wants to send packets over any\n",(0,i.kt)("inlineCode",{parentName:"p"},"terraX")," interface. The ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk-dhd.ko")," will relay them into the ",(0,i.kt)("inlineCode",{parentName:"p"},"AF_PACKET")," queue\nand make the queue socket FD readable, and that eventually results in VPP\ninvoking the node's ",(0,i.kt)("inlineCode",{parentName:"p"},"tg_link_local_rx()")," function. It will fetch all packets\nfrom the kernel, convert them into VPP buffers, mark them with the proper link\ninterface, and invoke TX on the real Wigig interface."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},' clib_file_main_t *fm = &file_main;\n clib_file_t template = {0};\n\n template.read_function = tg_link_local_rx_fd_read_ready;\n template.file_descriptor = wi.data_fd;\n template.description = format (0, "%s", "wigig-local-rx");\n template.private_data = vec_len (tm->wigig_devs);\n\n wdev.clib_file_index = clib_file_add (fm, &template);\n')),(0,i.kt)("h4",{id:"tg-wired-local-rx"},"tg-wired-local-rx"),(0,i.kt)("p",null,"This node intercepts packets received from the ",(0,i.kt)("inlineCode",{parentName:"p"},"nicX")," tap interface and sends\nthem directly over the corresponding wired port. It is similar in function to\n",(0,i.kt)("inlineCode",{parentName:"p"}
1,"tg-link-local-rx"),", but for wired interfaces."),(0,i.kt)("p",null,"Instantiated in the ",(0,i.kt)("inlineCode",{parentName:"p"},"tg_wired_interface_enable()")," function in the plugin."),(0,i.kt)("h2",{id:"hqos-scheduler-tghqos"},"HQoS Scheduler (TGHQoS)"),(0,i.kt)("p",null,"Terragraph uses a custom Hierarchical Quality-of-Service (HQoS) scheduler on\ntraffic transmitted out of Wigig interfaces for managing prioritization of\ndifferent traffic types. The implementation is originally based on ",(0,i.kt)("a",{parentName:"p",href:"https://wiki.fd.io/view/VPP"},"VPP"),"'s\nnow-deprecated HQoS functionality that used the ",(0,i.kt)("a",{parentName:"p",href:"http://doc.dpdk.org/guides/prog_guide/qos_framework.html"},"DPDK QoS framework")," implemented\nin DPDK's ",(0,i.kt)("inlineCode",{parentName:"p"},"librte_sched")," library."),(0,i.kt)("h3",{id:"code-structure-1"},"Code Structure"),(0,i.kt)("p",null,"The HQoS implementation exists as a separate module within VPP's DPDK plugin. It\nis included as a series of patches in\n",(0,i.kt)("inlineCode",{parentName:"p"},"meta-qoriq/recipes-extended/vpp/vpp_19.09-lsdk.bb"),", with code added into the\ndirectory ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp/src/plugins/dpdk/tghqos"),"."),(0,i.kt)("h3",{id:"hierarchy"},"Hierarchy"),(0,i.kt)("p",null,"The HQoS scheduler has 4 hierarchical levels, consisting of (1) port, (2) pipe,\n(3) traffic class, (4) queue. A port is a Wigig device, and a pipe is a Wigig\npeer. Packet IPv6 DSCP headers are used to classify packets into different\ntraffic classes with 3 different colors. Currently Terragraph uses 4 traffic\nclasses with 1 queue per traffic class. The color is coded as the drop\nprecedence of the packet in accordance with RFC 2597."),(0,i.kt)("h3",{id:"red-dropper"},"RED Dropper"),(0,i.kt)("p",null,"To avoid congestion, packets arriving at the scheduler go through a dropper that\nsupports Random Early Detection (RED), Weighted Random Early Detection (WRED),\nand tail drop algorithms. The RED dropper is the same as the original one in\n",(0,i.kt)("inlineCode",{parentName:"p"}
1,"librte_sched"),", ported into VPP to decouple the HQoS implementation from any\nspecific DPDK libraries."),(0,i.kt)("h3",{id:"cli-commands"},"CLI Commands"),(0,i.kt)("p",null,"The HQoS scheduler has many associated CLI commands for configuration, testing,\nand stat collection purposes. The HQoS-specific CLI commands are provided in and\ndocumented in ",(0,i.kt)("inlineCode",{parentName:"p"},"tghqos_cli.c"),", including example command invocations and outputs."),(0,i.kt)("h3",{id:"hqos-operation"},"HQoS Operation"),(0,i.kt)("p",null,"The HQoS scheduler is logically part of the VPP Wigig output interface. When a\npacket is directed to a Wigig interface for tx (",(0,i.kt)("inlineCode",{parentName:"p"},"WigigX/Y/Z/0-tx"),"), it will\nfirst go through the HQoS scheduler before being passsed to the driver for\ntransmission."),(0,i.kt)("p",null,"The HQoS scheduler performs classification, enqueue, and dequeue operations on\neach packet. For classification, the scheduler reads the mbuf's link id metadata\nstored in a ",(0,i.kt)("inlineCode",{parentName:"p"},"dynfield")," of the mbuf to determine the pipe to which the packet is\nassigned, and it maps the DSCP value to the appropriate traffic class, queue,\nand color. This information gets stored in the mbuf's ",(0,i.kt)("inlineCode",{parentName:"p"},"sched")," field. The enqueue\noperation uses this information to store the packet in the appropriate queue,\ndropping the packet as the RED dropper algorithm deems necessary or if the queue\nis full. The dequeue operation currently uses strict priority scheduling by\ndefault, where packets are dequeued from the HQoS scheduler in strict priority\norder according to traffic class. Since the Wigig driver has separate transmit\nrings for each peer, the HQoS scheduler uses feedback from the driver about each\ntransmit ring's occupancy to determine how many packets to dequeue for each\npipe. Note that the traffic class prioritization is thus applied within each\npipe independently, and not across all pipes at once. After packets are dequeued\nfrom the HQoS scheduler, they are passed to the driver for transmission."),(0,i.kt)("p",null,"Weighted round robin scheduling with priority levels is also provided as an\noption. Traffic classes are assigned a priority level and a weight. Traffic\nclasses at a higher priority level are served before traffic classes at a lower\npriority level. Traffic classes at the same priority level are served in\naccordance to assigned weight. Each traffic class must have a non-zero weight\nin one priority level. Besides that requirement, weights can be arbitrary\nnon-negative integers; for each traffic class the scheduler will use the\nproportion of its weight to the sum of weights in that priority level to\ndetermine the proportion of packet segments that traffic class can send during\neach transmission period."),(0,i.kt)("p",null,"By default, the HQoS operations for a Wigig device run in the VPP node graph on\nthe same worker thread used for its rx processing, but it can also be c
1onfigured\nto use separate threads (see\n",(0,i.kt)("a",{parentName:"p",href:"#vpp-implementation-interface-thread-assignment"},"Interface Thread Assignment"),")."),(0,i.kt)("h2",{id:"wigig-jumbo-frame-support"},"Wigig Jumbo Frame Support"),(0,i.kt)("p",null,"The Wigig device software has an MTU of 4000 bytes, so any packets of length\ngreater than 4000 are considered jumbo frames that must be segmented before\nbeing transmitted across a Wigig link."),(0,i.kt)("h3",{id:"vpp-and-dpdk-buffers"},"VPP and DPDK Buffers"),(0,i.kt)("p",null,"Packets are represented as DPDK ",(0,i.kt)("inlineCode",{parentName:"p"},"rte_mbuf")," structs, and each ",(0,i.kt)("inlineCode",{parentName:"p"},"rte_mbuf")," has 2176\nbytes of data space in VPP. Packets of total packet length greater than 2176 are\nrepresented as a chain of ",(0,i.kt)("inlineCode",{parentName:"p"},"rte_mbuf")," segments. Multi-segment packets are also\nchained with ",(0,i.kt)("inlineCode",{parentName:"p"},"vlib_buffer_t")," linkage for processing in VPP."),(0,i.kt)("p",{align:"center"},(0,i.kt)("img",{src:"/figures/vpp_buffers.png",width:"1000"})),(0,i.kt)("p",null,"After VPP receives packets via ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk-input"),", the vlib buffer's ",(0,i.kt)("inlineCode",{parentName:"p"},"current_data"),"\noffset pointer, which is used as the start of data to be processed in VPP, is\nset to ",(0,i.kt)("inlineCode",{parentName:"p"},"mbuf->data_off - 128"),"\n(",(0,i.kt)("inlineCode",{parentName:"p"},"RTE_PKTMBUF_HEADROOM = VLIB_BUFFER_PRE_DATA_SIZE = 128"),"):"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," b[0]->current_data = mb[0]->data_off - RTE_PKTMBUF_HEADROOM;\n")),(0,i.kt)("p",null,"This refers to the same address as ",(0,i.kt)("inlineCode",{parentName:"p"},"mbuf->buf_addr + mbuf->data_off"),"."),(0,i.kt)("h3",{id:"jumbo-frame-segmentation"},"Jumbo Frame Segmentation"),(0,i.kt)("p",null,"Jumbo frames can be easily segmented by cutting the mbuf linkage along the mbuf\nchain. After inserting a custom header, these segments can be enqueued for\ntransmission as individual packets."),(0,i.kt)("p",{align:"center"},(0,i.kt)("img",{src:"/figures/vpp_seg_header.png",width:"1000"})),(0,i.kt)("p",null,"Segmented Terragraph jumbo frame packets are identified by the custom Ethernet\ntype ",(0,i.kt)("inlineCode",{parentName:"p"},"0xFF71")," (",(0,i.kt)("inlineCode",{parentName:"p"},"TG_JUMBO"),"). Since segmented Terragraph packets exist only on a\nWigig segment between two TG nodes, the value of the Ethernet type itself is not\nimportant. The combination of the custom fields ",(0,i.kt)("inlineCode",{parentName:"p"},"packet_id"),", ",(0,i.kt)("inlineCode",{parentName:"p"},"seg_index"),", and\n",(0,i.kt)("inlineCode",{parentName:"p"},"last_seg")," are used in segmentation and reassembly. A ",(0,i.kt)("inlineCode",{parentName:"p"},"protocol_version")," field\nis included for futureproofing."),(0,i.kt)("p",null,"The current maximum supported jumbo frame packet size in VPP is about 18KB. This\nis determined by the number of segments that can be stored according to\n",(0,i.kt)("inlineCode",{parentName:"p"},"TG_JUMBO_FRAME_SEG_ARRAY_LEN"),"."),(0,i.kt)("h4",{id:"transmitting--segmentation"},"Transmitting / Segmentation"),(0,i.kt)("p",null,"Segmentation is done prior to the input of the HQoS transmit queue in the DPDK\nplugin. Jumbo frames are identified by packet length and are segmented by\nbreaking the mbuf chain. Custom segmentation headers are prepended to each\nsegment's packet data."),(0,i.kt)("p",null,"There is no guarantee that each segment has sufficient headroom to fit the\nsegmentation header, so packet data must be shifted to allow for enough space\nfor the header. Each ",(0,i.kt)("inlineCode",{parentName:"p"},"rte_mbuf")," would typically be allocated\n",(0,i.kt)("inlineCode",{parentName:"p"},"RTE_PKTMBUF_HEADROOM")," (128) bytes of headroom, but drivers may choose to leave\n0 bytes of headroom for noninitial segments (e.g. DPAA2 driver). Accordingly,\nsegments that require data shifting will likely not have enough tailroom, and\nwill require the tail end of its packet data to be copied to the next segment."),(0,i.kt)("p",null,"After cutting segment chains, shifting and copying data, and inserting headers,\nsegments are enqueued to the HQoS transmit queue as individual packets. No\nfurther VPP node processing will occur so there is no need to address\n",(0,i.kt)("inlineCode",{parentName:"p"},"vlib_buffer_t")," linkage."),(0,i.kt)("h4",{id:"receiving--reassembly"},"Receiving / Reassembly"),(0,i.kt)("p",null,"Reassembly is done as soon as packets are received from the driver in the DPDK\nplugin, prior to injecting the packet into the rest of the VPP node graph. Only\none fragmented packet can be in flight at a time, per flow. Since the Wigig has\nno concept of traffic class, there is a maximum of one fragmented packet per\npeer per sector."),(0,i.kt)("p",null,"After an array of mbufs is received from the driver, jumbo frame segment packets\nare identified by their Ethernet type and removed from the mbuf processing\narray. They are stored in an array per device according to the link id found in\na ",(0,i.kt)("inlineCode",{parentName:"p"},"dynfield")," of the mbuf and its ",(0,i.kt)("inlineCode",{parentName:"p"},"seg_index")," field. After a segment with the\n",(0,i.kt)("inlineCode",{parentName:"p"},"last_seg")," field set is received, reassembly is attempted by first checking that\nthere is a segment for every prior index, and verifying all stored segments have\nthe same ",(0,i.kt)("inlineCode",{parentName:"p"},"packet_id"),". Then, create the linkage for the packet by chaining the\nmbufs, and increment each mbuf segment's data_off field to skip over the custom\nsegmentation header."),(0,i.kt)("p",null,"The reassembled single packet is then passed on to ",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk_process_rx_burst")," for\nfurther processing, including creating ",(0,i.kt)("inlineCode",{parentName:"p"},"vlib_buffer_t")," linkage."),(0,i.kt)("h2",{id:"vpp-threads--core-allocation"},"VPP Threads & Core Allocation"),(0,i.kt)("p",null,"The table below lists all VPP and PMD threads along with their CPU core\nallocation on Puma hardware (4 cores)."),
1(0,i.kt)("table",null,(0,i.kt)("thead",{parentName:"table"},(0,i.kt)("tr",{parentName:"thead"},(0,i.kt)("th",{parentName:"tr",align:null},"Thread Name"),(0,i.kt)("th",{parentName:"tr",align:null},"Core #"),(0,i.kt)("th",{parentName:"tr",align:null},"Purpose"))),(0,i.kt)("tbody",{parentName:"table"},(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp_main")),(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},"Handles CLI, Wigig, and network events, etc.")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"vpp_wk_X")),(0,i.kt)("td",{parentName:"tr",align:null},"2-3"),(0,i.kt)("td",{parentName:"tr",align:null},"Datapath workers (one per CPU core)")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"eal-intr-thread")),(0,i.kt)("td",{parentName:"tr",align:null},"0"),(0,i.kt)("td",{parentName:"tr",align:null},"Primary PCI interrupt dispatcher")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"wil6210_wmiX")),(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},"Interrupt handlers (one per Wigig card)")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"poll-dhdX")),(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"ioctl")," dispatcher (one per Wigig card)")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"wil-fw-log-poll")),(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},"Firmware/ucode log polling (one per Wigig card)")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"nl60g-poll")),(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},"Communicate with ",(0,i.kt)("inlineCode",{parentName:"td"},"host_manager_11ad")," utility (one per Wigig card)")))),(0,i.kt)("p",null,"The core allocation is set by ",(0,i.kt)("inlineCode",{parentName:"p"},"update_vpp_startup_conf.lua")," using the VPP\nconfiguration options ",(0,i.kt)("inlineCode",{parentName:"p"},"cpu skip-cores"),", ",(0,i.kt)("inlineCode",{parentName:"p"},"cpu workers"),", and\n",(0,i.kt)("inlineCode",{parentName:"p"},"dpdk dev <pci_id> workers"),'. Threads created by the PMD are set to use the\nmaster lcore ("logical core") for DPDK, i.e. the core used for ',(0,i.kt)("inlineCode",{parentName:"p"},"vpp_main"),"."),(0,i.kt)("p",null,"On Puma, CPU cores 2 and 3 are reserved for the datapath, enforced by passing\n",(0,i.kt)("inlineCode",{parentName:"p"},"isolcpus=2,3")," in the Linux kernel configuration. Most user processes are\nexpected to run on CPU core 1, configured by runit via ",(0,i.kt)("inlineCode",{parentName:"p"},"taskset")," (refer to\n",(0,i.kt)("a",{parentName:"p",href:"/docs/developer/Service_Scripts#service-scripts-runit-scripts"},"runit scripts")," for more\ndetails). However, the remaining core is ",(0,i.kt)("em",{parentName:"p"},"not")," used for the datapath, as several\nfirmware crashes have been observed when reserving three datapath cores."),(0,i.kt)("a",{id:"vpp-implementation-interface-thread-assignment"}),(0,i.kt)("h3",{id:"interface-thread-assignment"},"Interface Thread Assignment"),(0,i.kt)("p",null,"When an interface is assigned to a worker thread, its rx processing begins on\nthat thread, and packets received on that interface continue through the VPP\nnode graph on that thread. When these packets are directed to tx interfaces\nwithout HQoS enabled, they are sent to the driver of the tx interface on the\nsame thread as it was received."),(0,i.kt)("p",null,"When these packets are directed to tx interfaces with HQoS enabled, packets are\nnot immediately sent to the driver, and the next immediate step is determined\nby whether there are separate HQoS threads enabled:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"No HQoS thread enabled")," - Packets are enqueued into HQoS queues. Packets\nare dequeued from HQoS queues and sent to the device driver during the\nexecution of the ",(0,i.kt)("inlineCode",{parentName:"li"},"dpdk-input")," polling node associated with the tx interface.\nThis happens on the worker thread to which its rx processing was assigned,\nnot the worker thread on which the packet was originally received."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"HQoS thread enabled")," - Packets are passed to the HQoS thread via a software\nqueue in VPP. The separate HQoS thread will receive these packets and handle\nenqueuing into HQoS queues, dequeueing from HQoS queues, and sending to device\ndrivers.")),(0,i.kt)("p",null,"The current default configuration has HQoS enabled on Wigig devices and no HQoS\nthreads enabled. HQoS threads can be enabled by using the\n",(0,i.kt)("inlineCode",{parentName:"p"}
1,"cpu corelist-tghqos-threads")," option in VPP's startup configuration and moving\nall interface rx processing onto one worker thread. Enabling HQoS threads may be\nbeneficial for certain types of traffic, particularly with small packet sizes on\nnodes with 1 or 2 Wigig sectors, but performance suffers when passing traffic at\nmaximum throughput on 3 or 4 sectors."),(0,i.kt)("h2",{id:"firmware-message-paths"},"Firmware Message Paths"),(0,i.kt)("p",null,"This section describes the flow of firmware ",(0,i.kt)("inlineCode",{parentName:"p"},"ioctl")," calls (initiated by\nuser-space programs) and firmware events (initiated by firmware) within VPP. For\na system-level view of these message paths, refer to\n",(0,i.kt)("a",{parentName:"p",href:"/docs/developer/Driver_Stack#driver-stack-example-message-path"},"Driver Stack"),"."),(0,i.kt)("h3",{id:"firmware-ioctl"},"Firmware ",(0,i.kt)("inlineCode",{parentName:"h3"},"ioctl")),(0,i.kt)("p",null,"Firmware ",(0,i.kt)("inlineCode",{parentName:"p"},"ioctl")," calls are initiated from user-space programs (e.g. e2e_minion)\nand require a completion notification from firmware. The path is as follows:"),(0,i.kt)("table",null,(0,i.kt)("thead",{parentName:"table"},(0,i.kt)("tr",{parentName:"thead"},(0,i.kt)("th",{parentName:"tr",align:null},"Sequence"),(0,i.kt)("th",{parentName:"tr",align:null},"Entity"),(0,i.kt)("th",{parentName:"tr",align:null},"Actions"))),(0,i.kt)("tbody",{parentName:"table"},(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"poll-dhdX")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on ",(0,i.kt)("inlineCode",{parentName:"td"},"dhdX")," network interface ",(0,i.kt)("inlineCode",{parentName:"td"},"wmi_send()"),(0,i.kt)("br",null),"Write ",(0,i.kt)("inlineCode",{parentName:"td"},"ioctl")," message to PCI")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"2"),(0,i.kt)("td",{parentName:"tr",align:null},"Wigig firmware"),(0,i.kt)("td",{parentName:"tr",align:null},"Process ",(0,i.kt)("inlineCode",{parentName:"td"},"ioctl")," and generate completion event")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"3"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"eal-intr-thread")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on ",(0,i.kt)("inlineCode",{parentName:"td"},"VFIO_MSI")," interrupt",(0,i.kt)("br",null),"Read firmware message from PCI",(0,i.kt)("br",null),"Dispatch to per-interface work thread")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"4"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"wil6210_wmiX")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on Linux futex (",(0,i.kt)("inlineCode",{parentName:"td"},"pthread_cond_wait()"),")",(0,i.kt)("br",null),"Read firmware message from work queue",(0,i.kt)("br",null),"Process completion event")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"5"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"poll-dhdX")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on Linux futex (",(0,i.kt)("inlineCode",{parentName:"td"},"pthread_cond_wait()"),")",(0,i.kt)("br",null),"Send completion event to driver")))),(0,i.kt)("h3",{id:"firmware-event"},"Firmware Event"),(0,i.kt)("p",null,"Firmware events are initiated by firmware and do ",(0,i.kt)("em",{parentName:"p"},"not")," require a completion\nnotification. The path is as follows:"),(0,i.kt)("table",null,(0,i.kt)("thead",{parentName:"table"},(0,i.kt)("tr",{parentName:"thead"},(0,i.kt)("th",{parentName:"tr",align:null},"Sequence"),(0,i.kt)("th",{parentName:"tr",align:null},"Entity"),(0,i.kt)("th",{parentName:"tr",align:null},"Actions"))),(0,i.kt)("tbody",{parentName:"table"},(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"1"),(0,i.kt)("td",{parentName:"tr",align:null},"Wigig firmware"),(0,i.kt)("td",{parentName:"tr",align:null},"Generate event")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"2"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"eal-intr-thread")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on ",(0,i.kt)("inlineCode",{parentName:"td"},"VFIO_MSI")," interrupt",(0,i.kt)("br",null),"Read firmware message from PCI",(0,i.kt)("br",null),"Dispatch to per-interface work thread")),(0,i.kt)("tr",{parentName:"tbody"},(0,i.kt)("td",{parentName:"tr",align:null},"3"),(0,i.kt)("td",{parentName:"tr",align:null},(0,i.kt)("inlineCode",{parentName:"td"},"wil6210_wmiX")),(0,i.kt)("td",{parentName:"tr",align:null},"Wake on Linux futex (",(0,i.kt)("inlineCode",{parentName:"td"},"pthread_cond_wait()"),")",(0,i.kt)("br",null),"Read firmware message from work queue",(0,i.kt)("br",null),"Send event to driver")))),(0,i.kt)("h2",{id:"debugging"},"Debugging"),(0,i.kt)("p",null,"This section briefly describes possible debugging options for VPP."),(0,i.kt)("h3",{id:"use-debug-builds"},"Use Debug Builds"),(0,i.kt)("p",null,"Recompile VPP and any plugins (ex. ",(0,i.kt)("inlineCode",{parentName:"p"},"vpp-tgcfg"),") with debug information by adding\nthese lines to ",(0,i.kt)("inlineCode",{parentName:"p"},"conf/local.conf"),":"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},'DEBUG_BUILD_pn-vpp = "1"\nDEBUG_BUILD_pn-vpp-tgcfg = "1"\n')),(0,i.kt)("h3",{id:"enable-core-dumps"},"Enable Core Dumps"),(0,i.kt)("p",null,"VPP core dumps can be enabled via the node configuration field\n",(0,i.kt)("inlineCode",{parentName:"p"},"envParams.VPP_COREDUMP_ENABLED")," (enabled by default). Core dumps are written to\n",(0,i.kt)("inlineCode",{parentName:"p"},"/var/volatile/cores/")," and can be loaded into ",(0,i.kt)("inlineCode",{parentName:"p"},"gdb"),"."),(0,i.kt)("h3",{id:"collect-packet-traces"},"Collect Packet Traces"),(0,i.kt)("p",null,'VPP allows collecting packet traces at different points of the datapath using\nthe "trace" commands (e.g. ',(0,i.kt)("inlineCode",{parentName:"p"},"trace add <node>"),", ",(0,i.kt)("inlineCode",{parentName:"p"},"show trace"),", ",(0,i.kt)("inlineCode",{parentName:"p"},"clear trace"),").\nSome important graph nodes include:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"dpdk-input"),": All traffic arriving into VPP from wired or wigig interfaces"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"tg-link-local-rx"),": All traffic arriving into VPP from ",(0,i.kt)("inlineCode",{parentName:"li"},"terraX")," interfaces on\nthe Linux side"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"tapcli-rx"),": All traffic arriving into VPP from the ",(0,i.kt)("inlineCode",{parentName:"li"},"vnet0")," interface on the\nLinux side"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"vpp-terraX-tx"),": All traffic generated in VPP for ",(0,i.kt)("inlineCode",{parentName:"li"},"vpp-terraX")," interfaces\n(replace X with the actual index)"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"pg-trace"),": All traffic generated by VPP's packet generator")),(0,i.kt)("h2",{id:"resources"},"Resources"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"https://www.dpdk.org/"},"DPDK")," - Data Plane Development Kit"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"http://doc.dpdk.org/guides/prog_guide/qos_framework.html"},"DPDK QoS Framework")," - DPDK QoS Framework implemented in ",(0,i.kt)("inlineCode",{parentName:"li"},"librte_sched")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"https://wiki.fd.io/view/VPP"},"VPP")," - Vector Packet Processing"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"https://github.com/facebook/openr"},"Open/R")," - Meta's routing platform")))}c.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.