https://terragraph.com/assets/js/0823eadd.83e9086c.js

1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[3803],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>g});var n=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function l(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),d=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},p=function(e){var t=d(e.components);return n.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),m=d(a),g=r,c=m["".concat(s,".").concat(g)]||m[g]||u[g]||i;return a?n.createElement(c,o(o({ref:t},p),{},{components:a})):n.createElement(c,o({ref:t},p))}));function g(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,o=new Array(i);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var d=2;d<i;d++)o[d]=a[d];return n.createElement.apply(null,o)}return n.createElement.apply(null,a)}m.displayName="MDXCreateElement"},6085:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>u,frontMatter:()=>i,metadata:()=>l,toc:()=>d});var n=a(7462),r=(a(7294),a(3905));const i={},o="Software Upgrade",l={unversionedId:"developer/Software_Upgrade",id:"developer/Software_Upgrade",title:"Software Upgrade",description:"This document describes the in-band software upgrade procedure for Terragraph",source:"@site/../docs/developer/Software_Upgrade.md",sourceDirName:"developer",slug:"/developer/Software_Upgrade",permalink:"/docs/developer/Software_Upgrade",draft:!1,editUrl:"https://github.com/terragraph/meta-terragraph/edit/main/docs/../docs/developer/Software_Upgrade.md",tags:[],version:"current",frontMatter:{},sidebar:"developerManualSidebar",previous:{title:"Network Ignition",permalink:"/docs/developer/Network_Ignition"},next:{title:"Configuration Management",permalink:"/docs/developer/Configuration_Management"}},s={},d=[{value:"Upgrade Stages",id:"upgrade-stages",level:2},{value:"Parallelization",id:"parallelization",level:2},{value:"&quot;Prepare&quot; Stage",id:"prepare-stage",level:3},{value:"&quot;Commit&quot; Stage",id:"commit-stage",level:3},{value:"Step 1: Identify Articulation Points",id:"step-1-identify-articulation-points",level:4},{value:"Step 2: Select Leaf Sites",id:"step-2-select-leaf-sites",level:4},{value:"Mixed Hardware Upgrades",id:"mixed-hardware-upgrades",level:2},{value:"Golden Images",id:"golden-images",level:2},{value:"Message Interface",id:"message-interface",level:2},{value:"Upgrade Procedure",id:"upgrade-procedure",level:3},{value:"Software Image Management",id:"software-image-management",level:3},{value:"Resources",id:"resources",level:2}],p={toc:d};

1function u(e){let{components:t,...a}=e;return(0,r.kt)("wrapper",(0,n.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"software-upgrade"},"Software Upgrade"),(0,r.kt)("p",null,"This document describes the in-band software upgrade procedure for Terragraph\nnodes."),(0,r.kt)("h2",{id:"upgrade-stages"},"Upgrade Stages"),(0,r.kt)("p",null,"The controller's ",(0,r.kt)("inlineCode",{parentName:"p"},"UpgradeApp")," manages in-band software upgrades. An upgrade is\ndivided into two stages:"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("strong",{parentName:"li"},"Prepare"),": A new software image is distributed to nodes in the network.\nEach minion downloads the image, verifies its integrity, and flashes it to\nits secondary disk partition."),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("strong",{parentName:"li"},"Commit"),": Nodes are instructed to switch to the new software image. Each\nminion swaps its primary and secondary disk partitions and performs a full\nreboot. This stage is disruptive, and may affect other elements in the\nnetwork.")),(0,r.kt)("h2",{id:"parallelization"},"Parallelization"),(0,r.kt)("p",null,"Within each upgrade stage, commands are executed across multiple nodes in\nparallel. The specifics are given in the sections below."),(0,r.kt)("h3",{id:"prepare-stage"},'"Prepare" Stage'),(0,r.kt)("p",null,"Software images are distributed over BitTorrent by default. The controller seeds\neach image through its own BitTorrent tracker, and sends the magnet URIs to each\nminion to download. Alternatively, images can be distributed over HTTP or HTTPS\nby sending the web URL instead; the images must be hosted via an external HTTP\nserver (ex. nginx in the default Docker Swarm deployment)."),(0,r.kt)("p",null,'The distribution of software images is fully parallelizable, with the only\nconcern being high bandwidth utilization. This can be limited by setting speed\nlimits in each BitTorrent client, or by limiting the number of nodes downloading\nthe image at any point in time ("batching").'),(0,r.kt)("p",null,'When using BitTorrent, both the controller and minion will publish the following\nstats during the "prepare" stage:'),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Key"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.progressPpm")),(0,r.kt)("td",{parentName:"tr",align:null},"The torrent download progress in parts per million, i.e. ",(0,r.kt)("inlineCode",{parentName:"td"},"[0,1000000]"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.seeds")),(0,r.kt)("td",{parentName:"tr",align:null},"The number of connected peers that are seeding the torrent")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.peers")),(0,r.kt)("td",{parentName:"tr",align:null},"The number of connected peers")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.connections")),(0,r.kt)("td",{parentName:"tr",align:null},"The number of peer connections, including half-open connections")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.downloadBps")),(0,r.kt)("td",{parentName:"tr",align:null},"The total download rate for all peers (in bps)")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.uploadBps")),(0,r.kt)("td",{parentName:"tr",align:null},"The total upload rate for all peers (in bps)")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.downloadPayloadBytes")),(0,r.kt)("td",{parentName:"tr",align:null},"The numer of payload bytes received during this session (ignoring protocol overhead)")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"upgrade.bt.uploadPayloadBytes")),(0,r.kt)("td",{parentName:"tr",align:null},"The numer of payload bytes sent during this session (ignoring protocol overhead)")))),(0,r.kt)("h3",{id:"commit-stage"},'"Commit" Stage'),(0,r.kt)("p",null,'The commit of new images is more difficult to parallelize, as nodes must reboot\nto apply the new image. A reboot brings down all of a node\'s wireless and\nEthernet links, potentially disconnecting parts of the network ("network\nisolation"). A bad sequence of reboots could repeatedly isolate the same\nportions of the network.'),(0,r.kt)("p",null,"The commit parallelization algorithm is designed as follows:"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},"Avoid network isolation. If upgrading a node must cause network\nisolation (due to poor network design), then upgrade the isolated portion of\nthe network at the same time."),(0,r.kt)("li",{parentName:"ol"},"Upgrade entire sites at once. This simplifies the graph logic, and aligns\nwith future DN hardware (i.e. single nodes with multiple radios)."),(0,r.kt)("li",{parentName:"ol"},"Commit sites that are one hop away from another reachable site (when\npossible). This reduces the amount of time required to re-ignite sites.")),(0,r.kt)("p",null,"The parallelization algorithm is a two-step procedure, and contained within\n",(0,r.kt)("inlineCode",{parentName:"p"},"GraphHelper"),". The steps are described below."),(0,r.kt)("h4",{id:"step-1-identify-articulation-points"},"Step 1: Identify Articulation Points"),(0,r.kt)("p",null,"In a connected graph, a vertex is called an ",(0,r.kt)("em",{parentName:"p"},(0,r.kt)("a",{parentName:"em",href:"https://www.geeksforgeeks.org/articulation-points-or-cut-vertices-in-a-graph/"},"articulation point"))," (AP) if\nremoving it would result in a disconnected graph. In the commit algorithm, APs\nare the sites that will isolate portions of the network if taken down or removed\nfrom the topology."),(0,r.kt)("p",null,"In the example below, site F is identified as an AP because taking it down would\nisolate sites H and G."),(0,r.kt)("p",{align:"center"},(0,r.kt)("img",{src:"/figures/upgrade_ap.svg",width:"750"})),(0,r.kt)("p",null,'After identifying all APs, the list is filtered further to remove any "sub-APs"\n(i.e. APs isolated by other APs). Each AP forms an "AP group", consisting of\nitself and all sites it isolates. All sites in an AP group will be updated\nsimultaneously in the same batch.'),(0,r.kt)("p",null,"In the previous example, sites F, H, and G form an AP group and are treated as a\nsingle virtual site."),(0,r.kt)("h4",{id:"step-2-select-leaf-sites"},"Step 2: Select Leaf Sites"),(0,r.kt)("p",null,"Once all AP groups are identified, each site or AP group is now accessible via\nmultiple paths. The graph is then converted into a spanning tree using a custom\nspanning tree decomposition algorithm, which is a modified DFS with a bias\ntoward previously-upgraded sites. This will influence previously-upgraded sites\nto form the core of the tree, rather than the edge."),(0,r.kt)("p",null,'Leaf sites in the resulting spanning tree can be upgraded simultaneously (in a\n"batch") without causing any disruption to other sites in the spanning tree. The\nspanning tree algorithm is repeated until each site is picked once, resulting in\nseveral "batches".'),(0,r.kt)("p",null,"Using the same example from above, the following illustration shows this step of\nthe algorithm, which requires three iterations to perform a full network\nupgrade."),(0,r.kt)("p",{align:"center"},(0,r.kt)("img",{src:"/figures/upgrade_tree.svg",width:"750"})),(0,r.kt)("p",null,"The controller sends commit requests to all nodes in an upgrade batch. Once the\noperation is complete, the two-step algorithm is run again to identify the next\nbatch. The algorithm always uses the current network state when computing the\nnext batch, accounting for any topology changes that might have occurred between\nsteps."),(0,r.kt)("h2",{id:"mixed-hardware-upgrades"},"Mixed Hardware Upgrades"),(0,r.kt)("p",null,"Different node hardware may require different software images;

1 the list of\nsupported hardware board IDs is written in each image's metadata section\n(",(0,r.kt)("inlineCode",{parentName:"p"},"thrift::ImageMeta::hardwareBoardIds"),"). The following procedure is used to\nupgrade a network with mixed node hardware:"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},'Issue a "prepare" request for each software image.'),(0,r.kt)("li",{parentName:"ol"},'Issue a single "commit" request for all nodes.')),(0,r.kt)("h2",{id:"golden-images"},"Golden Images"),(0,r.kt)("p",null,'The controller can be configured to automatically upgrade nodes to a "golden\nimage" through the controller configuration field ',(0,r.kt)("inlineCode",{parentName:"p"},"upgradeParams.goldenImage"),'.\nWhen enabled, the controller periodically looks for nodes running software\nversions older than the golden image (by comparing major/minor numbers only)\nand schedules "prepare" and "commit" procedures back-to-back for these nodes.'),(0,r.kt)("h2",{id:"message-interface"},"Message Interface"),(0,r.kt)("p",null,"All controller messages related to upgrades are described below."),(0,r.kt)("h3",{id:"upgrade-procedure"},"Upgrade Procedure"),(0,r.kt)("p",null,"The following commands address the upgrade procedure itself:"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"User Operation"),(0,r.kt)("th",{parentName:"tr",align:null},"Command"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Send Upgrade Request"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_GROUP_REQ"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Abort Upgrade"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_ABORT_REQ"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Get Upgrade State"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_STATE_REQ"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Get Upgrade Commit Plan"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_COMMIT_PLAN_REQ"))))),(0,r.kt)("p",null,"All types of upgrades are initiated via the ",(0,r.kt)("inlineCode",{parentName:"p"},"UpgradeGroupReq")," structure, which\ncontains the nodes to upgrade and a nested ",(0,r.kt)("inlineCode",{parentName:"p"},"UpgradeReq")," structure that defines\nthe actual upgrade procedure. The type of upgrade is determined by the\n",(0,r.kt)("inlineCode",{parentName:"p"},"thrift::UpgradeReqType")," enum:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"PREPARE_UPGRADE"),' - The "prepare" stage (described above).'),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"COMMIT_UPGRADE"),' - The "commit" stage (described above).'),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"FULL_UPGRADE"),' - Execute the "prepare" and "commit" stages back-to-back.'),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"RESET_STATUS")," - Reset each node's local upgrade state, and cancel any ongoing\nimage downloads or scheduled commits.")),(0,r.kt)("p",null,"All upgrade requests are forwarded from the controller to minions via the\n",(0,r.kt)("inlineCode",{parentName:"p"},"UPGRADE_REQ")," message."),(0,r.kt)("h3",{id:"software-image-management"},"Software Image Management"),(0,r.kt)("p",null,"The following commands manage the software images hosted and seeded by the\ncontroller:"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"User Operation"),(0,r.kt)("th",{parentName:"tr",align:null},"Command"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Add Upgrade Image"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_ADD_IMAGE_REQ"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Delete Upgrade Image"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_DEL_IMAGE_REQ"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"List Upgrade Images"),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UPGRADE_LIST_IMAGES_REQ"))))),(0,r.kt)("p",null,"The ",(0,r.kt)("inlineCode",{parentName:"p"},"thrift::UpgradeImage")," structure contains the URIs for downloading the\nsoftware images:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"magnetUri")," - The magnet URI (i.e. for BitTorrent)"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"httpUri")," - The HTTP URI (only set if HTTP serving is enabled)")),(0,r.kt)("h2",{id:"resources"},"Resources"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("a",{parentName:"li",href:"https://www.geeksforgeeks.org/articulation-points-or-cut-vertices-in-a-graph/"},"articulation point")," - Articulation point algorithms")))}u.isMDXComponent=!0}}]);