1"use strict";(self.webpackChunklearn_v_2=self.webpackChunklearn_v_2||[]).push([["15300"],{12587(e,n,i){i.r(n),i.d(n,{metadata:()=>t,default:()=>h,frontMatter:()=>l,contentTitle:()=>r,toc:()=>c,assets:()=>o});var t=JSON.parse('{"id":"Alerts & Notifications/Notifications/Agent Dispatched Notifications/Agent Notifications Reference","title":"Agent Alert Notifications","description":"Netdata\'s Agent can send alert notifications directly from each node. It supports a wide range of services, multiple recipients, and role-based routing.","source":"@site/docs/Alerts & Notifications/Notifications/Agent Dispatched Notifications/Agent Notifications Reference.mdx","sourceDirName":"Alerts & Notifications/Notifications/Agent Dispatched Notifications","slug":"/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference","permalink":"/docs/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference","draft":false,"unlisted":false,"editUrl":"https://github.com/netdata/netdata/edit/master/src/health/notifications/README.md","tags":[],"version":"current","lastUpdatedAt":1784546513000,"sidebarPosition":10,"frontMatter":{"custom_edit_url":"https://github.com/netdata/netdata/edit/master/src/health/notifications/README.md","sidebar_label":"Agent Notifications Reference","learn_status":"Published","learn_rel_path":"Alerts & Notifications/Notifications/Agent Dispatched Notifications","sidebar_position":10,"learn_link":"https://learn.netdata.cloud/docs/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference","slug":"/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference"},"sidebar":"sidebar","previous":{"title":"Agent Dispatched Notifications","permalink":"/docs/alerts-&-notifications/notifications/agent-dispatched-notifications"},"next":{"title":"Alerta","permalink":"/docs/alerts-&-notifications/notifications/agent-dispatched-notifications/alerta"}}'),s=i(74848),a=i(28453);let l={custom_edit_url:"https://github.com/netdata/netdata/edit/master/src/health/notifications/README.md",sidebar_label:"Agent Notifications Reference",learn_status:"Published",learn_rel_path:"Alerts & Notifications/Notifications/Agent Dispatched Notifications",sidebar_position:"10",learn_link:"https://learn.netdata.cloud/docs/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference",slug:"/alerts-&-notifications/notifications/agent-dispatched-notifications/agent-notifications-reference"},r="Agent Alert Notifications",o={},c=[{value:"How It Works",id:"how-it-works",level:2},{value:"Role-Based Routing Visualization",id:"role-based-routing-visualization",level:2},{value:"Health Management API Workflow",id:"health-management-api-workflow",level:2},{value:"Quick Setup",id:"quick-setup",level:2},{value:"Example: Alert with Role-Based Routing",id:"example-alert-with-role-based-routing",level:2},{value:"Advanced Role-Based Routing Examples",id:"advanced-role-based-routing-examples",level:2},{value:"Health Management API",id:"health-management-api",level:2},{value:"API Authorization",id:"api-authorization",level:3},{value:"Common API Commands",id:"common-api-commands",level:3},{value:"Configuration Options",id:"configuration-options",level:2},{value:"Testing Your Notification Setup",id:"testing-your-notification-setup",level:2},{value:"Debugging with Trace",id:"debugging-with-trace",level:3},{value:"Troubleshooting Alert Notifications",id:"troubleshooting-alert-notifications",level:2},{value:"Email Notifications Not Working",id:"email-notifications-not-working",level:3},{value:"Slack Notifications Failing",id:"slack-notifications-failing",level:3},{value:"PagerDuty Integration Issues",id:"pagerduty-integration-issues",level:3},{value:"Notification Delays",id:"notification-delays",level:3},{value:"Related Docs",id:"related-docs",level:2}];function d(e){let n={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",mermaid:"mermaid",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,a.R)(),...e.components},{Details:i}=n;return i||function(e,n){throw Error("Expected "+(n?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}("Details",!0),(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"agent-alert-notifications",children:"Agent Alert Notifications"})}),"\n",(0,s.jsx)(n.p,{children:"Netdata's Agent can send alert notifications directly from each node. It supports a wide range of services, multiple recipients, and role-based routing."}),"\n",(0,s.jsx)(n.h2,{id:"how-it-works",children:"How It Works"}),"\n",(0,s.jsxs)(n.p,{children:["The Agent uses a notification script defined in ",(0,s.jsx)(n.code,{children:"netdata.conf"})," under the ",(0,s.jsx)(n.code,{children:"[health]"})," section:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:"script to execute on alarm = /usr/libexec/netdata/plugins.d/alarm-notify.sh\n"})}),"\n",(0,s.jsxs)(n.p,{children:["The default script is ",(0,s.jsx)(n.code,{children:"alarm-notify.sh"}),"."]}),"\n",(0,s.jsx)(n.p,{children:"This script handles:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:"Multiple recipients"}),"\n",(0,s.jsx)(n.li,{children:"Multiple notification methods"}),"\n",(0,s.jsxs)(n.li,{children:["Role-based routing (e.g., ",(0,s.jsx)(n.code,{children:"sysadmin"}),", ",(0,s.jsx)(n.code,{children:"webmaster"}),", ",(0,s.jsx)(n.code,{children:"dba"}),")"]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"role-based-routing-visualization",children:"Role-Based Routing Visualization"}),"\n",(0,s.jsx)(n.mermaid,{value:'flowchart TD\n Alert("High CPU Usage Alert") --\x3e Check("Severity Level")\n \n Check --\x3e|"WARNING"| WarningRouting("Role: SysAdmin")\n Check --\x3e|"CRITICAL"| CriticalRouting("Multiple Roles")\n \n WarningRouting --\x3e SlackChannel
1("Slack")\n WarningRouting --\x3e EmailOps("Email")\n \n CriticalRouting --\x3e PagerDuty("PagerDuty")\n CriticalRouting --\x3e EmailManagers("Email")\n CriticalRouting --\x3e SlackUrgent("Slack")\n CriticalRouting --\x3e SMS("SMS")\n \n %% Style definitions\n classDef alert fill:#ffeb3b,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef neutral fill:#f9f9f9,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef complete fill:#4caf50,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef database fill:#2196F3,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n\n %% Apply styles\n class Alert alert\n class Check database\n class WarningRouting,CriticalRouting complete\n class SlackChannel,EmailOps,PagerDuty,EmailManagers,SlackUrgent,SMS neutral'}),"\n",(0,s.jsx)(n.h2,{id:"health-management-api-workflow",children:"Health Management API Workflow"}),"\n",(0,s.jsx)(n.mermaid,{value:'flowchart TD\n Start("Normal Operation") --\x3e Maintenance("Maintenance Window?")\n \n Maintenance --\x3e|"No"| NormalOps("Continue Normal Alerting")\n Maintenance --\x3e|"Yes"| ApiAction("Choose Action")\n \n ApiAction --\x3e SilenceAll("SILENCE ALL")\n ApiAction --\x3e DisableAll("DISABLE ALL")\n ApiAction --\x3e SilenceSelect("SILENCE Specific")\n \n SilenceAll --\x3e Reset("RESET when done")\n DisableAll --\x3e Reset\n SilenceSelect --\x3e Reset\n \n Reset --\x3e Restored("Normal Operations Restored")\n \n %% Style definitions\n classDef alert fill:#ffeb3b,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef neutral fill:#f9f9f9,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef complete fill:#4caf50,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n classDef database fill:#2196F3,stroke:#000000,stroke-width:3px,color:#000000,font-size:18px\n\n %% Apply styles\n class Start,Restored complete\n class Maintenance database\n class ApiAction alert\n class NormalOps,SilenceAll,DisableAll,SilenceSelect,Reset neutral'}),"\n",(0,s.jsx)(n.h2,{id:"quick-setup",children:"Quick Setup"}),"\n",(0,s.jsx)(n.admonition,{type:"tip",children:(0,s.jsxs)(n.p,{children:["Use the ",(0,s.jsx)(n.code,{children:"edit-config"})," script to safely edit configuration files. It automatically creates the necessary files in the right place and opens them in your editor.\n",(0,s.jsxs)(n.a,{href:"/docs/netdata-agent/configuration#edit-configuration-files",children:["Learn how to use ",(0,s.jsx)(n.code,{children:"edit-config"})]})]})}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Open the Agent's health notification config:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"sudo ./edit-config health_alarm_notify.conf\n"})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Set up the required API keys or credentials for the service you want to use."}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:["Define recipients per ",(0,s.jsx)(n.strong,{children:"role"})," (see below)."]}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Changes take effect immediately. The next alarm will use the updated configuration, so no restart or reload is required."}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"example-alert-with-role-based-routing",children:"Example: Alert with Role-Based Routing"}),"\n",(0,s.jsxs)(n.p,{children:["Here's an example alert assigned to the ",(0,s.jsx)(n.code,{children:"sysadmin"})," role from the ",(0,s.jsx)(n.code,{children:"ram.conf"})," file:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:"alarm: ram_in_use\n on: system.ram\nclass: Utilization\n type: System\ncomponent: Memory\n os: linux\n hosts: *\n calc: $used * 100 / ($used + $cached + $free + $buffers)\n units: %\n every: 10s\n warn: $this > (($status >= $WARNING) ? (80) : (90))\n crit: $this > (($status == $CRITICAL) ? (90) : (98))\n delay: down 15m multiplier 1.5 max 1h\n info: system memory utilization\n to:
1sysadmin\n"})}),"\n",(0,s.jsxs)(n.p,{children:["Then, in ",(0,s.jsx)(n.code,{children:"health_alarm_notify.conf"}),", you assign recipients per notification method:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected] [email protected]"\nrole_recipients_slack[sysadmin]="#alerts #infra"\n'})}),"\n",(0,s.jsx)(n.h2,{id:"advanced-role-based-routing-examples",children:"Advanced Role-Based Routing Examples"}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"DevOps Team Example"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'# Backend team receives database and application server alerts\nrole_recipients_slack[backend]="#backend-team"\nrole_recipients_pagerduty[backend]="PDK3Y5EXAMPLE"\n\n# Frontend team receives web server and CDN alerts\nrole_recipients_slack[frontend]="#frontend-team"\nrole_recipients_opsgenie[frontend]="key1example"\n\n# Security team receives all security-related alerts\nrole_recipients_email[security]="[email protected]"\nrole_recipients_slack[security]="#security-alerts"\n\n# SRE team receives critical infrastructure alerts 24/7\nrole_recipients_slack[sre]="#sre-alerts"\nrole_recipients_pagerduty[sre]="PDK3Y5SREXAMPLE"\nrole_recipients_telegram[sre]="123456789"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Time-Based Routing Example"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"You can use external scripts to dynamically change recipients based on work hours, on-call schedules, etc.:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'# Use a script to determine the current on-call engineer\nONCALL_EMAIL=$(get_oncall_email.sh)\nrole_recipients_email[oncall]="${ONCALL_EMAIL}"\nrole_recipients_sms[oncall]="${ONCALL_PHONE}"\n\n# Standard business hours team gets non-critical alerts during work hours\nrole_recipients_slack[business_hours]="#daytime-monitoring"\n'})})]}),"\n",(0,s.jsx)(n.h2,{id:"health-management-api",children:"Health Management API"}),"\n",(0,s.jsx)(n.p,{children:"Netdata provides a powerful Health Management API that lets you control alert behavior during maintenance windows, testing, or other planned activities."}),"\n",(0,s.jsx)(n.h3,{id:"api-authorization",children:"API Authorization"}),"\n",(0,s.jsxs)(n.p,{children:["The API is protected by an authorization token stored in ",(0,s.jsx)(n.code,{children:"/var/lib/netdata/netdata.api.key"}),":"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'# Get your token\nTOKEN=$(cat /var/lib/netdata/netdata.api.key)\n\n# Use the token in API calls\ncurl "http://localhost:19999/api/v1/manage/health?cmd=RESET" -H "X-Auth-Token: ${TOKEN}"\n'})}),"\n",(0,s.jsx)(n.h3,{id:"common-api-commands",children:"Common API Commands"}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Disable All Health Checks"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Completely stops evaluation of health checks during maintenance:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl "http://localhost:19999/api/v1/manage/health?cmd=DISABLE ALL" -H "X-Auth-Token: ${TOKEN}"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Silence All Notifications"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Continues to evaluate health checks but prevents notifications:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl "http://localhost:19999/api/v1/manage/health?cmd=SILENCE ALL" -H "X-Auth-Token: ${TOKEN}"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Disable Specific Alerts"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Target only certain alerts by name, chart, context, host, or family:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'# Silence all disk space alerts\ncurl "http://localhost:19999/api/v1/manage/health?cmd=SILENCE&context=disk_space" -H "X-Auth-Token: ${TOKEN}"\n\n# Disable CPU alerts for specific hosts\ncurl "http://localhost:19999/api/v1/manage/health?cmd=DISABLE&context=cpu&hosts=prod-db-*" -H "X-Auth-Token: ${TOKEN}"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"View Current Silenced/Disabled Alerts"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Check what's currently silenced or disabled:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl "http://localhost:19999/api/v1/manage/health?cmd=LIST" -H "X-Auth-Token: ${TOKEN}"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Reset to Normal Operation"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Re-enable all health checks and notifications:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl "http://localhost:19999/api/v1/manage/health?cmd=RESET" -H "X-Auth-Token: ${TOKEN}"\n'})})]}),"\n",(0,s.jsx)(n.h2,{id:"configuration-options",children:"Configuration Options"}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Recipients Per Role"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Define who receives alerts and how:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected]"\nrole_recipients_telegram[webmaster]="123456789"\nrole_recipients_slack[dba]="#database-alerts"\n'})}),(0,s.jsx)(n.p,{children:"Use spaces to separate multiple recipients."}),(0,s.jsx)(n.p,{children:"To disable a notification method for a role, use:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="disabled"\
1n'})}),(0,s.jsx)(n.p,{children:"If left empty, the default recipient for that method is used."})]}),"\n",(0,s.jsxs)(i,{id:"alert-severity-filtering",children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Alert Severity Filtering"})}),(0,s.jsx)("br",{}),(0,s.jsxs)(n.p,{children:["You can limit certain recipients to only receive ",(0,s.jsx)(n.strong,{children:"critical"})," alerts:"]}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected] [email protected]|critical"\n'})}),(0,s.jsx)(n.p,{children:"This setup:"}),(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Sends all alerts to ",(0,s.jsx)(n.code,{children:"[email protected]"})]}),"\n",(0,s.jsxs)(n.li,{children:["Sends notifications to ",(0,s.jsx)(n.code,{children:"[email protected]"})," only once the alarm reaches CRITICAL, then continues sending status changes (including WARNING and CLEAR) until the alarm is cleared."]}),"\n"]}),(0,s.jsx)(n.p,{children:"Works for all supported methods: email, Slack, Telegram, Twilio, Discord, etc."})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Controlling Recovered (CLEAR) Notifications"})}),(0,s.jsx)("br",{}),(0,s.jsxs)(n.p,{children:["When an alert returns to normal, Netdata sends a ",(0,s.jsx)(n.strong,{children:"CLEAR"})," (recovered) notification. You can control when and whether these are sent."]}),(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Default behavior:"})," Netdata suppresses CLEAR notifications when the alert was never in a WARNING or CRITICAL state. If ",(0,s.jsx)(n.code,{children:"old_status"})," was not WARNING or CRITICAL and the alert transitions to CLEAR, no notification is sent. This prevents noise from alerts that flap without ever reaching a problem state."]}),(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Enable CLEAR for all transitions:"})," If your downstream system handles deduplication, set ",(0,s.jsx)(n.code,{children:"clear_alarm_always"})," in ",(0,s.jsx)(n.code,{children:"health_alarm_notify.conf"})," to override the default suppression and send a CLEAR notification regardless of the previous status:"]}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:"clear_alarm_always='YES'\n"})}),(0,s.jsxs)(n.p,{children:[(0,s.jsxs)(n.strong,{children:["Filter by CRITICAL history with the ",(0,s.jsx)(n.code,{children:"|critical"})," modifier:"]})," As described in ",(0,s.jsx)(n.a,{href:"#alert-severity-filtering",children:"Alert Severity Filtering"})," above, ",(0,s.jsx)(n.code,{children:"|critical"})," forwards notifications only for alerts that have reached CRITICAL status. This affects both WARNING and CLEAR:"]}),(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"WARNING"})," notifications are suppressed unless the alarm has previously reached CRITICAL."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"CLEAR"})," notifications are only sent when the alert previously passed through CRITICAL. If the alert only went through WARNING \u2192 CLEAR, the CLEAR is not forwarded."]}),"\n"]}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected]|critical"\n'})}),(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Suppress all CLEAR notifications:"})," Use the ",(0,s.jsx)(n.code,{children:"|noclear"})," modifier to completely block CLEAR notifications for a recipient while still receiving WARNING and CRITICAL alerts:"]}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected]|noclear"\n'})}),(0,s.jsx)(n.p,{children:"You can combine modifiers. This example notifies only for alarms that have reached CRITICAL (WARNING is suppressed until then), and excludes CLEAR notifications entirely:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'role_recipients_email[sysadmin]="[email protected]|critical|noclear"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Proxy Settings"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"To send notifications via a proxy, set these environment variables:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'export http_proxy="http://10.0.0.1:3128/"\
1nexport https_proxy="http://10.0.0.1:3128/"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Notification Images"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"By default, Netdata includes public image URLs in notifications (hosted by the global Registry)."}),(0,s.jsx)(n.p,{children:"To use custom image paths:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'images_base_url="http://my.public.netdata.server:19999"\n'})})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Custom Date Format"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"Change the timestamp format in notifications:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:'date_format="+%F %T%:z" # Example: RFC 3339\n'})}),(0,s.jsx)(n.p,{children:"Common formats:"}),(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Format"}),(0,s.jsx)(n.th,{children:"String"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"ISO 8601"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"+%FT%T%z"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"RFC 5322"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"+%a, %d %b %Y %H:%M:%S %z"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"RFC 3339"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"+%F %T%:z"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"Local time"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"+%x %X"})})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:"ANSI C / asctime()"}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.em,{children:"(leave empty)"})})]})]})]}),(0,s.jsxs)(n.p,{children:["See ",(0,s.jsx)(n.code,{children:"man date"})," for more formatting options."]})]}),"\n",(0,s.jsxs)(i,{children:[(0,s.jsx)("summary",{children:(0,s.jsx)("strong",{children:"Hostname Format"})}),(0,s.jsx)("br",{}),(0,s.jsx)(n.p,{children:"By default, Netdata uses the short hostname in notifications."}),(0,s.jsx)(n.p,{children:"To use the fully qualified domain name (FQDN), set:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-ini",children:"use_fqdn=YES\n"})}),(0,s.jsxs)(n.p,{children:["If you've set a custom hostname in ",(0,s.jsx)(n.code,{children:"netdata.conf"}),", that value takes priority."]})]}),"\n",(0,s.jsx)(n.h2,{id:"testing-your-notification-setup",children:"Testing Your Notification Setup"}),"\n",(0,s.jsx)(n.p,{children:"You can test alert notifications manually."}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'# Switch to the Netdata user\nsudo su -s /bin/bash netdata\n\n# Enable debugging\nexport NETDATA_ALARM_NOTIFY_DEBUG=1\n\n# Test default role (sysadmin)\n./plugins.d/alarm-notify.sh test\n\n# Test specific role\n./plugins.d/alarm-notify.sh test "webmaster"\n'})}),"\n",(0,s.jsxs)(n.admonition,{type:"important",children:[(0,s.jsx)(n.p,{children:"If you're running your own Netdata Registry, set:"}),(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'export NETDATA_REGISTRY_URL="https://your.registry.url"\n'})}),(0,s.jsx)(n.p,{children:"before testing."})]}),"\n",(0,s.jsx)(n.h3,{id:"debugging-with-trace",children:"Debugging with Trace"}),"\n",(0,s.jsx)(n.p,{children:"To see the full execution output:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"bash -x ./plugins.d/alarm-notify.sh test\n"})}),"\n",(0,s.jsx)(n.p,{children:"Then look for the internal calls and re-run the one you want to trace in more detail."}),"\n",(0,s.jsx)(n.h2,{id:"troubleshooting-alert-notifications",children:"Troubleshooting Alert Notifications"}),"\n",(0,s.jsx)(n.p,{children:"Here are solutions for common alert notification issues:"}),"\n",(0,s.jsx)(n.h3,{id:"email-notifications-not-working",children:"Email Notifications Not Working"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Verify your email configuration:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'grep -E "SEND_EMAIL|DEFAULT_RECIPIENT_EMAIL" /etc/netdata/health_alarm_notify.conf\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Check if the system can send mail:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'echo "Test" | mail -s "Test Email" [email protected]\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Look for errors in the Netdata log:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'tail -f /var/log/netdata/error.log | grep "alarm notify"\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Test with debugging enabled:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"sudo su -s /bin/bash netdata\nexport NETDATA_ALARM_NOTIFY_DEBUG=1\n./plugins.d/alarm-notify.sh test\n"})}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"slack-notifications-failing",children:"Slack Notifications Failing"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Verify your webhook URL is correct:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'grep -E "SLACK_WEBHOOK_URL" /etc/netdata/health_alarm_notify.conf\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Check for network connectivity to Slack:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl -X POST -H "Content-type: application/json" --data \'{"text":"Test"}\' YOUR_WEBHOOK_URL\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:["Confirm channel names start with ",(0,s.jsx)(n.code,{children:"#"})," in your configuration."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"pagerduty-integration-issues",children:"PagerDuty Integration Issues"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Verify your service key:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'grep -E "DEFAULT_RECIPIENT_PD|SEND_PD" /etc/netdata/health_alarm_notify.conf\n'})}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsx)(n.p,{children:"Test the PagerDuty API directly:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:'curl -H "Content-Type: application/json" -X POST -d \'{"service_key":"YOUR_SERVICE_KEY","event_type":"trigger","description":"Test"}\' https://events.pagerduty.com/generic/2010-04-15/create_event.json\n'})}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h3,{id:"notification-delays",children:"Notification Delays"}),"\n",(0,s.jsx)(n.p,{children:"If notifications seem delayed:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:["Check the ",(0,s.jsx)(n.code,{children:"delay"})," parameter in your alarm definitions in ",(0,s.jsx)(n.code,{children:"health.d/*.conf"}
1)," \u2014 this is the only per-alert delay setting for notifications; there is no global delay setting in ",(0,s.jsx)(n.code,{children:"health_alarm_notify.conf"}),"."]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"related-docs",children:"Related Docs"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"/docs/alerts-&-notifications/alert-configuration-reference",children:"How to configure alerts"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"/docs/alerts-&-notifications/notifications#notification-methods",children:"Notification methods list"})}),"\n",(0,s.jsx)(n.li,{children:(0,s.jsx)(n.a,{href:"/docs/netdata-agent/configuration",children:"Netdata configuration basics"})}),"\n"]})]})}function h(e={}){let{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},28453(e,n,i){i.d(n,{R:()=>l,x:()=>r});var t=i(96540);let s={},a=t.createContext(s);function l(e){let n=t.useContext(a);return t.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:l(e.components),t.createElement(a.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.