https://near-nodes.io/assets/js/af359ccf.b2fbc1bf.js

1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[4103],{3905:(e,n,t)=>{t.d(n,{Zo:()=>p,kt:()=>E});var a=t(7294);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function i(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?o(Object(t),!0).forEach((function(n){r(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):o(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function l(e,n){if(null==e)return{};var t,a,r=function(e,n){if(null==e)return{};var t,a,r={},o=Object.keys(e);for(a=0;a<o.length;a++)t=o[a],n.indexOf(t)>=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)t=o[a],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var s=a.createContext({}),d=function(e){var n=a.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},p=function(e){var n=d(e.components);return a.createElement(s.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},c=a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),c=d(t),E=r,m=c["".concat(s,".").concat(E)]||c[E]||u[E]||o;return t?a.createElement(m,i(i({ref:n},p),{},{components:t})):a.createElement(m,i({ref:n},p))}));function E(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var o=t.length,i=new Array(o);i[0]=c;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var d=2;d<o;d++)i[d]=t[d];return a.createElement.apply(null,i)}return a.createElement.apply(null,t)}c.displayName="MDXCreateElement"},4715:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>s,contentTitle:()=>i,default:()=>u,frontMatter:()=>o,metadata:()=>l,toc:()=>d});var a=t(7462),r=(t(7294),t(3905));const o={id:"upgrade_alert",title:"Validator Node Upgrade Alert",sidebar_label:"Validator Node Upgrade Alert",description:"How to setup an alert for validator nodes upgrading.",sidebar_position:5},i="Alerting for Validator Node Upgrades",l={unversionedId:"maintenance/upgrade_alert",id:"maintenance/upgrade_alert",title:"Validator Node Upgrade Alert",description:"How to setup an alert for validator nodes upgrading.",source:"@site/../docs/maintenance/upgrade_alert.md",sourceDirName:"maintenance",slug:"/maintenance/upgrade_alert",permalink:"/maintenance/upgrade_alert",draft:!1,editUrl:"https://github.com/near/node-docs/edit/main/website/../docs/maintenance/upgrade_alert.md",tags:[],version:"current",lastUpdatedBy:"Darioush Jalali",lastUpdatedAt:1773760174,formattedLastUpdatedAt:"Mar 17, 2026",sidebarPosition:5,frontMatter:{id:"upgrade_alert",title:"Validator Node Upgrade Alert",sidebar_label:"Validator Node Upgrade Alert",description:"How to setup an alert for validator nodes upgrading.",sidebar_position:5},sidebar:"tutorialSidebar",previous:{title:"Safe Database Migrations",permalink:"/maintenance/db-migrations"},next:{title:"Submit Feature Request and Bug Report",permalink:"/bug-report"}},s={},d=[],p={toc:d};

1function u(e){let{components:n,...t}=e;return(0,r.kt)("wrapper",(0,a.Z)({},p,t,{components:n,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"alerting-for-validator-node-upgrades"},"Alerting for Validator Node Upgrades"),(0,r.kt)("p",null,"Please note that once 80% of the validator nodes switch to a new protocol version, the upgrade will occur in 2 epochs. Any validator node who doesn't upgrade in time will be kicked. The following provides a network upgrade ratio which allows validators to see what percentage of the validator nodes has upgraded to a new protocol version."),(0,r.kt)("blockquote",{class:"warning"},(0,r.kt)("strong",null,"Heads up"),(0,r.kt)("br",null),(0,r.kt)("br",null),(0,r.kt)("p",null,"Information about node version and build is available on-chain. A metric based on on-chain data would be more reliable, and that metric is already under construction.")),(0,r.kt)("p",null,"Validator nodes periodically submit telemetry data, which is stored in a publicly-accessible PostgresSQL database.\nNote that the nodes can opt out of submitting the telemetry data, and the telemetry data isn't guaranteed to be accurate."),(0,r.kt)("p",null,"With these caveats clarified, let's define an upgrade alert. The instructions were tested only in Grafana."),(0,r.kt)("p",null,"Step 1. Add a PostgreSQL data source with the following credentials:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"telemetry_testnet")," for testnet and ",(0,r.kt)("inlineCode",{parentName:"li"},"telemetry_mainnet")," for mainnet: ",(0,r.kt)("a",{parentName:"li",href:"https://github.com/near/near-indexer-for-explorer/#shared-public-access"},"https://github.com/near/near-indexer-for-explorer/#shared-public-access"))),(0,r.kt)("p",null,"Step 2. Add a dashboard with a Graph panel with the following SQL query. Grafana only supports alerts on the Graph panels, therefore this needs a workaround to fit the Table data into a Graph format."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},"SELECT $__time(time_sec)\n  ,upgraded_ratio\nFROM (\n  SELECT NOW() - INTERVAL '1' SECOND AS time_sec\n    ,(\n      (\n        SELECT CAST(COUNT(*) AS FLOAT)\n        FROM nodes\n        WHERE agent_version > (\n            SELECT MIN(agent_version)\n            FROM nodes\n            WHERE is_validator\n              AND (now() - last_seen) < INTERVAL '15 MINUTE'\n            )\n          AND is_validator\n          AND (now() - last_seen) < INTERVAL '15 MINUTE'\n        ) / (\n        SELECT COUNT(*)\n        FROM nodes\n        WHERE is_validator\n          AND (now() - last_seen) < INTERVAL '15 MINUTE'\n        )\n      ) AS upgraded_ratio\n  ) AS inner_table\nWHERE $__timeFilter(time_sec);\n")),(0,r.kt)("p",null,"Step 3. Go to the alert tab and change the condition to ",(0,r.kt)("inlineCode",{parentName:"p"},"WHEN last () OF query (A, 10s, now) IS ABOVE 0.65")," or similar."),(0,r.kt)("p",null,"Step 4. Optionally add a table with min and max version of the validator nodes:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},"SELECT (\n    SELECT MIN(agent_version)\n    FROM nodes\n    WHERE is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS min_version\n  ,(\n    SELECT MAX(agent_version)\n    FROM nodes\n    WHERE is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS max_version\n  ,(\n    SELECT COUNT(*)\n    FROM nodes\n    WHERE is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS num_validators\n  ,(\n    SELECT COUNT(*)\n    FROM nodes\n    WHERE agent_version = (\n        SELECT MAX(agent_version)\n        FROM nodes\n        WHERE is_validator\n          AND (now() - last_seen) < INTERVAL '15 MINUTE'\n        )\n      AND is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS upgraded_validators\n  ,(\n    (\n      SELECT CAST(COUNT(*) AS FLOAT)\n      FROM nodes\n      WHERE agent_version = (\n          SELECT MAX(agent_version)\n          FROM nodes\n          WHERE is_validator\n            AND (now() - last_seen) < INTERVAL '15 MINUTE'\n          )\n        AND is_validator\n        AND (now() - last_seen) < INTERVAL '15 MINUTE'\n      ) / (\n      SELECT COUNT(*)\n      FROM nodes\n      WHERE is_validator\n        AND (now() - last_seen) < INTERVAL '15 MINUTE'\n      )\n    ) AS upgraded_ratio;\n")),(0,r.kt)("p",null,"Step 5. Optionally add a table with the number of nodes running a newer version than your node:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"}

1,"SELECT (\n    SELECT COUNT(*)\n    FROM nodes\n    WHERE is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS num_validators\n  ,(\n    SELECT COUNT(*)\n    FROM nodes\n    WHERE agent_version > (\n        SELECT agent_version\n        FROM nodes\n        WHERE moniker = '$YOUR_VALIDATOR_MONIKER'\n        )\n      AND is_validator\n      AND (now() - last_seen) < INTERVAL '15 MINUTE'\n    ) AS upgraded_validators\n  ,(\n    (\n      SELECT CAST(COUNT(*) AS FLOAT)\n      FROM nodes\n      WHERE agent_version > (\n          SELECT agent_version\n          FROM nodes\n          WHERE moniker = '$YOUR_VALIDATOR_MONIKER'\n          )\n        AND is_validator\n        AND (now() - last_seen) < INTERVAL '15 MINUTE'\n      ) / (\n      SELECT COUNT(*)\n      FROM nodes\n      WHERE is_validator\n        AND (now() - last_seen) < INTERVAL '15 MINUTE'\n      )\n    ) AS upgraded_ratio;\n")),(0,r.kt)("blockquote",null,(0,r.kt)("p",{parentName:"blockquote"},"Got a question?"),(0,r.kt)("a",{href:"https://stackoverflow.com/questions/tagged/nearprotocol"},(0,r.kt)("h8",null,"Ask it on StackOverflow!"))))}u.isMDXComponent=!0}}]);