https://docs.rspamd.com/assets/js/bd3f995c.f784be8e.js

1"use strict";(self.webpackChunkrspamd_docs=self.webpackChunkrspamd_docs||[]).push([[4612],{28453:(e,n,s)=>{s.d(n,{R:()=>l,x:()=>c});var i=s(96540);const t={},r=i.createContext(t);function l(e){const n=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function c(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:l(e.components),i.createElement(r.Provider,{value:n},e.children)}},46960:(e,n,s)=>{s.r(n),s.d(n,{assets:()=>o,contentTitle:()=>c,default:()=>h,frontMatter:()=>l,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"modules/bayes_expiry","title":"Bayes expiry module","description":"The bayes expiry module provides intelligent expiration of statistical tokens for Redis-based Bayesian classifiers using the new schema storage format.","source":"@site/docs/modules/bayes_expiry.md","sourceDirName":"modules","slug":"/modules/bayes_expiry","permalink":"/modules/bayes_expiry","draft":false,"unlisted":false,"editUrl":"https://github.com/rspamd/docs.rspamd.com/edit/master/docs/modules/bayes_expiry.md","tags":[],"version":"current","frontMatter":{"title":"Bayes expiry module"},"sidebar":"docs","previous":{"title":"ASN module","permalink":"/modules/asn"},"next":{"title":"Chartable module","permalink":"/modules/chartable"}}');var t=s(74848),r=s(28453);const l={title:"Bayes expiry module"},c="Bayes expiry module",o={},d=[{value:"Overview",id:"overview",level:2},{value:"Classifier configuration",id:"classifier-configuration",level:2},{value:"Expire option values",id:"expire-option-values",level:3},{value:"Module configuration",id:"module-configuration",level:2},{value:"Cluster configuration",id:"cluster-configuration",level:3},{value:"Principles of operation",id:"principles-of-operation",level:2},{value:"Token classification",id:"token-classification",level:2},{value:"Expiration behavior",id:"expiration-behavior",level:2},{value:"Advantages",id:"advantages",level:3},{value:"Changing expire value",id:"changing-expire-value",level:3},{value:"Limiting memory usage",id:"limiting-memory-usage",level:2},{value:"Classifier configuration",id:"classifier-configuration-1",level:3},{value:"Redis configuration",id:"redis-configuration",level:3},{value:"Multi-class support",id:"multi-class-support",level:2},{value:"Monitoring",id:"monitoring",level:2}];function a(e){const n={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,r.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.header,{children:(0,t.jsx)(n.h1,{id:"bayes-expiry-module",children:"Bayes expiry module"})}),"\n",(0,t.jsxs)(n.p,{children:["The ",(0,t.jsx)(n.code,{children:"bayes expiry"})," module provides intelligent expiration of statistical tokens for Redis-based Bayesian classifiers using the ",(0,t.jsx)(n.code,{children:"new schema"})," storage format."]}),"\n",(0,t.jsx)(n.h2,{id:"overview",children:"Overview"}),"\n",(0,t.jsx)(n.p,{children:"The module automatically manages token lifetimes based on their statistical significance:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Significant tokens"})," (strongly associated with spam or ham) are kept permanently"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Common tokens"})," (appear equally in both classes) have reduced TTL"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Infrequent/insignificant tokens"})," expire according to the configured TTL"]}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"This ensures that valuable statistical data is preserved while less useful tokens are eventually purged."}),"\n",(0,t.jsx)(n.h2,{id:"classifier-configuration",children:"Classifier configuration"}),"\n",(0,t.jsxs)(n.p,{children:["Classifier settings go in ",(0,t.jsx)(n.code,{children:"/etc/rspamd/local.d/classifier-bayes.conf"}),":"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-hcl",children:"# Required: enable new schema (default since 2.0)\nnew_schema = true;\n\n# Token expiry time (seconds, or -1 for persistent, or false to disable)\nexpire = 8640000;  # ~100 days\n"})}),"\n",(0,t.jsx)(n.h3,{id:"expire-option-values",children:"Expire option values"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,t.jsxs)(n.table,{children:[(0,t.jsx)(n.thead,{children:(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.th,{children:"Value"}),(0,t.jsx)(n.th,{children:"Behavior"})]})}),(0,t.jsxs)(n.tbody,{children:[(0,t.jsxs)(n.tr,{children:[(0,t.jsxs)(n.td,{children:[(0,t.jsx)(n.code,{children:"N"})," (seconds)"]}),(0,t.jsx)(n.td,{children:"Set TTL to N seconds for non-significant tokens. Max: 2147483647"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"-1"})}),(0,t.jsx)(n.td,{children:"Make tokens persistent (no expiration)"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"false"})}),(0,t.jsx)(n.td,{children:"Disable bayes expiry for this classifier"})]})]})]}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"Note:"})," Setting ",(0,t.jsx)(n.code,{children:"expire = false"})," does not change existing token TTLs; only newly learned tokens will be persistent."]}),"\n",(0,t.jsx)(n.h2,{id:"module-configuration",children:"Module configuration"}),"\n",(0,t.jsxs)(n.p,{children:["Global module settings go in ",(0,t.jsx)(n.code,{children:"/etc/rspamd/local.d/bayes_expiry.conf"}),":"]}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,t.jsxs)(n.table,{children:[(0,t.jsx)(n.thead,{children:(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.th,{children:"Option"}),(0,t.jsx)(n.th,{children:"Type"}),(0,t.jsx)(n.th,{children:"Default"}),(0,t.jsx)(n.th,{children:"Description"})]})}),(0,t.jsxs)(n.tbody,{children:[(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"interval"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"60"})}),(0,t.jsx)(n.td,{children:"Seconds between expiry steps"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"count"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"1000"})}),(0,t.jsx)(n.td,{children:"Number of keys to check per step"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"epsilon_common"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"0.01"})}),(0,t.jsx)(n.td,{children:'Tolerance for classifying tokens as "common"'})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"common_ttl"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"864000"})}),(0,t.jsx)(n.td,{children:"TTL for common tokens (10 days)"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"significant_factor"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"0.75"})}),(0,t.jsx)(n.td,{children:"Threshold for token significance (75%)"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"cluster_nodes"})}),(0,t.jsx)(n.td,{children:"number"}),(0,t.jsx)(n.td,{children:(0,t.jsx)(n.code,{children:"0"})}),(0,t.jsx)(n.td,{children:"Number of cluster nodes (auto-detected from neighbours)"})]})]})]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-hcl",children:"# local.d/bayes_expiry.conf\ninterval = 90;\ncount = 15000;\n"})}),"\n",(0,t.jsx)(n.h3,{id:"cluster-configuration",children:"Cluster configuration"}),"\n",(0,t.jsxs)(n.p,{children:["In a clustered setup, the module automatically detects the number of neighbour nodes and adjusts the expiry interval to prevent multiple nodes from performing expiry simultaneously. You can override this with ",(0,t.jsx)(n.code,{children:"cluster_nodes"}),"."]}),"\n",(0,t.jsx)(n.h2,{id:"principles-of-operation",children:"Principles of operation"}),"\n",(0,t.jsx)(n.p,{children:"The module runs on the primary controller worker and performs expiry steps at regular intervals (default: every 60 seconds). Each step:"}),"\n",(0,t.jsxs)(n.ol,{children:["\n",(0,t.jsx)(n.li,{children:"Scans approximately 1000 tokens using Redis SCAN"}),"\n",(0,t.jsx)(n.li,{children:"Analyzes each token's occurrence frequency across classes"}),"\n",(0,t.jsx)(n.li,{children:"Adjusts TTLs based on token classification"}),"\n",(0,t.jsx)(n.li,{children:"Continues from where the previous step stopped"}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"A full iteration through all tokens depends on database size. For 10 million tokens, expect approximately one week per complete cycle."}),"\n",(0,t.jsx)(n.h2,{id:"token-classification",children:"Token classification"}),"\n",(0,t.jsx)(n.p,{children:"Tokens are categorized based on their occurrence patterns:"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,t.jsxs)(n.table,{children:[(0,t.jsx)(n.thead,{children:(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.th,{children:"Category"}),(0,t.jsx)(n.th,{children:"Description"}),(0,t.jsx)(n.th,{children:"Action"})]})}),(0,t.jsxs)(n.tbody,{children:[(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.strong,{children:"Significant"})}),(0,t.jsx)(n.td,{children:"Strongly associated with one class (>75% of occurrences)"}),(0,t.jsx)(n.td,{children:"Made persistent"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.strong,{children:"Common"})}),(0,t.jsxs)(n.td,{children:["No single class dominates: ",(0,t.jsx)(n.code,{children:"max_class_count / total <= (1 - epsilon_common)"})]}),(0,t.jsx)(n.td,{children:"TTL reduced to 10 days"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.strong,{children:"Insignificant"})}),(0,t.jsx)(n.td,{children:"Between significant and common"}),(0,t.jsx)(n.td,{children:"TTL set to expire value"})]}),(0,t.jsxs)(n.tr,{children:[(0,t.jsx)(n.td,{children:(0,t.jsx)(n.strong,{children:"Infrequent"})}),(0,t.jsx)(n.td,{children:"Very low total occurrences"}),(0,t.jsx)(n.td,{children:"TTL set to expire value"})]})]})]}),"\n",(0,t.jsx)(n.h2,{id:"expiration-behavior",children:"Expiration behavior"}),"\n",(0,t.jsx)(n.p,{children:'Since Rspamd 2.0, the module operates in "lazy" mode:'}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Significant tokens"}),": Set to persistent (TTL = -1) if they have a TTL"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Insignificant/infrequent tokens"}),": TTL reduced to ",(0,t.jsx)(n.code,{children:"expire"})," value if current TTL exceeds it"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.strong,{children:"Common tokens"}),": TTL reduced to ",(0,t.jsx)(n.code,{children:"common_ttl"})," (10 days) if current TTL exceeds it"]}),"\n"]}),"\n",(0,t.jsx)(n.h3,{id:"advantages",children:"Advantages"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"Statistics can be stored offline indefinitely without losing significant tokens"}),"\n",(0,t.jsx)(n.li,{children:"Minimizes unnecessary TTL updates"}),"\n",(0,t.jsx)(n.li,{children:"Simple backup: just copy the RDB file"}),"\n"]}),"\n",(0,t.jsx)(n.h3,{id:"changing-expire-value",children:"Changing expire value"}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"Decreasing expire"}),": TTLs exceeding the new value will be updated during the next cycle."]}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"Increasing expire"}),": First set ",(0,t.jsx)(n.code,{children:"expire = -1"})," and wait for one complete cycle to make tokens persistent, then set the new expire value."]}),"\n",(0,t.jsx)(n.h2,{id:"limiting-memory-usage",children:"Limiting memory usage"}),"\n",(0,t.jsx)(n.p,{children:"Use Redis memory limits with eviction to cap statistics storage:"}),"\n",(0,t.jsx)(n.h3,{id:"classifier-configuration-1",children:"Classifier configuration"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-hcl",children:'# local.d/classifier-bayes.conf\nbackend = "redis";\nservers = "localhost:6378";\nnew_schema = true;\nexpire = 2144448000;  # ~68 years (effectively never expires)\n'})}),"\n",(0,t.jsx)(n.h3,{id:"redis-configuration",children:"Redis configuration"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-sh",children:"# /etc/redis/redis-bayes.conf\ninclude /etc/redis/redis.conf\n\nport 6378\npidfile /var/run/redis/bayes.pid\nlogfile /var/log/redis/bayes.log\ndbfilename bayes.rdb\ndir /var/db/redis/bayes/\n\nmaxmemory 500MB\nmaxmemory-policy volatile-ttl\n"})}),"\n",(0,t.jsxs)(n.p,{children:["With ",(0,t.jsx)(n.code,{children:"volatile-ttl"})," eviction policy, Redis evicts keys with shorter TTLs first when memory limit is reached. Since significant tokens are persistent (no TTL), they're never evicted. Less important tokens with TTLs will be evicted as needed."]}),"\n",(0,t.jsxs)(n.p,{children:[(0,t.jsx)(n.strong,{children:"Important:"})," For this to work correctly, store Bayesian statistics in a separate Redis instance. See the ",(0,t.jsx)(n.a,{href:"/tutorials/redis_replication",children:"Redis replication"})," tutorial for multi-instance setup."]}),"\n",(0,t.jsx)(n.h2,{id:"multi-class-support",children:"Multi-class support"}),"\n",(0,t.jsx)(n.p,{children:"The module supports classifiers with more than two classes (not just spam/ham). Token significance is evaluated across all configured classes, with tokens being considered significant if they strongly associate with any single class."}),"\n",(0,t.jsxs)(n.p,{children:["The common-token test is also multi-class aware. A token is classified as common when no single class accounts for more than ",(0,t.jsx)(n.code,{children:"(1 - epsilon_common)"})," of its total occurrences \u2014 i.e. when ",(0,t.jsx)(n.code,{children:"max_class_count / total <= (1 - epsilon_common)"}),". With the default ",(0,t.jsx)(n.code,{children:"epsilon_common = 0.01"})," this means a token is common unless one class holds more than 99% of its counts. For the standard two-class (spam/ham) case this is equivalent to the token not being dominated by either class."]}),"\n",(0,t.jsxs)(n.p,{children:["Class label mappings can be provided via ",(0,t.jsx)(n.code,{children:"class_labels"})," in the classifier configuration. The module reads this mapping and stores it alongside the class symbols so that custom label names are correctly associated with their Redis keys."]}),"\n",(0,t.jsx)(n.h2,{id:"monitoring",children:"Monitoring"}),"\n",(0,t.jsx)(n.p,{children:"The module logs statistics after each step and complete cycle:"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{children:"finished expiry step 42: 1000 items checked, 150 significant (5 made persistent), \n50 insignificant (30 ttls set), 200 common (10 discriminated), \n600 infrequent (400 ttls set), 3.5 mean, 2.1 std\n"})}),"\n",(0,t.jsx)(n.p,{children:"At the end of each complete cycle, token occurrence distributions are also logged for each class."})]})}function h(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(a,{...e})}):a(e)}}}]);