https://docs.rspamd.com/assets/js/dedd0599.878f1965.js

1"use strict";(self.webpackChunkrspamd_docs=self.webpackChunkrspamd_docs||[]).push([[433],{19815:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>o,default:()=>u,frontMatter:()=>i,metadata:()=>s,toc:()=>d});const s=JSON.parse('{"id":"modules/elastic","title":"Elasticsearch module","description":"Elasticsearch module pushes a variety of message-related metadata to an instance of Elasticsearch or OpenSearch.","source":"@site/docs/modules/elastic.md","sourceDirName":"modules","slug":"/modules/elastic","permalink":"/modules/elastic","draft":false,"unlisted":false,"editUrl":"https://github.com/rspamd/docs.rspamd.com/edit/master/docs/modules/elastic.md","tags":[],"version":"current","frontMatter":{"title":"Elasticsearch module"},"sidebar":"docs","previous":{"title":"DMARC module","permalink":"/modules/dmarc"},"next":{"title":"Emails module","permalink":"/modules/emails"}}');var a=t(74848),r=t(28453);const i={title:"Elasticsearch module"},o="Elasticsearch module",l={},d=[{value:"Requirements",id:"requirements",level:2},{value:"Configuration",id:"configuration",level:2}];function c(e){const n={a:"a",code:"code",em:"em",h1:"h1",h2:"h2",header:"header",li:"li",p:"p",pre:"pre",ul:"ul",...(0,r.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.header,{children:(0,a.jsx)(n.h1,{id:"elasticsearch-module",children:"Elasticsearch module"})}),"\n",(0,a.jsxs)(n.p,{children:["Elasticsearch module pushes a variety of message-related metadata to an instance of ",(0,a.jsx)(n.a,{href:"https://elastic.co/",children:"Elasticsearch"})," or ",(0,a.jsx)(n.a,{href:"https://opensearch.org/",children:"OpenSearch"}),"."]}),"\n",(0,a.jsx)(n.p,{children:"Additionally module manages index template & policy and ingest pipeline for geoip functionality."}),"\n",(0,a.jsx)(n.h2,{id:"requirements",children:"Requirements"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsxs)(n.li,{children:["Supported version of ",(0,a.jsx)(n.a,{href:"https://www.elastic.co/",children:"Elasticsearch"})," or ",(0,a.jsx)(n.a,{href:"https://opensearch.org/",children:"OpenSearch"})," - Indexing database"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"https://www.elastic.co/products/kibana",children:"Kibana"})," or ",(0,a.jsx)(n.a,{href:"https://opensearch.org/",children:"OpenSearch Dashboards"})," (optional) - Used for data visualization"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"configuration",children:"Configuration"}),"\n",(0,a.jsxs)(n.p,{children:["The module is enabled by default (",(0,a.jsx)(n.code,{children:"enabled = true"}),"). It will self-disable at startup if no ",(0,a.jsx)(n.code,{children:"server"}),"/",(0,a.jsx)(n.code,{children:"servers"})," are configured, or if ",(0,a.jsx)(n.code,{children:"enabled = false"})," is set explicitly in ",(0,a.jsx)(n.code,{children:"local.d/elastic.conf"})," or ",(0,a.jsx)(n.code,{children:"override.d/elastic.conf"}),"."]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.em,{children:"Important:"})," by default module configures ",(0,a.jsx)(n.code,{children:"index_policy"})," to delete logs older then 30 days.\nIf you are updating from version 3.10.x or older and want to use a different index policy - please configure it before enabling this module."]}),"\n",(0,a.jsxs)(n.p,{children:["By default, the module automatically detects the distribution and whether the server version is supported, this behaviour can be disabled by setting ",(0,a.jsx)(n.code,{children:"autodetect_enabled"})," to ",(0,a.jsx)(n.code,{children:"false"}),", then it will take version of distribution from configuration."]}),"\n",(0,a.jsxs)(n.p,{children:["Automatic index template managment as well as index policy and geoip pipeline can be turned off by setting ",(0,a.jsx)(n.code,{children:"managed"})," to ",(0,a.jsx)(n.code,{children:"false"})," in corresponding config section."]}),"\n",(0,a.jsxs)(n.p,{children:["If you want to use your own existing index policy but keep a managed index template - you can set index policy ",(0,a.jsx)(n.code,{children:"managed"})," to ",(0,a.jsx)(n.code,{children:"false"})," and change the ",(0,a.jsx)(n.code,{children:"name"})," of the policy to your custom one."]}),"\n",(0,a.jsxs)(n.p,{children:["If you don't want to use index policy at all you need disable it by setting ",(0,a.jsx)(n.code,{children:"enabled"})," to ",(0,a.jsx)(n.code,{children:"false"})," in corresponding config section, same applies for geoip."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-hcl",children:'enabled = true;\nserver = "localhost:9200";\nuser = "elastic";\npassword = "elastic";\nuse_https = true;\nuse_gzip = true; # enable compression for requests\nuse_keepalive = true; # use HTTP keepalive connections\nallow_local = false; # store data for local/rspamc scans\nperiodic_interval = 5.0; # how often try to run background periodic tasks\ntimeout = 5.0; # how much wait for reply from elastic\nno_ssl_verify = false;\nversion = {\n  autodetect_enabled = true;\n  autodetect_max_fail = 30;\n  # override works only if autodetect is disabled; defaults shown below\n  override = {\n    name = "opensearch"; # default\n    version = "2.17"; # default\n  }\n};\nlimits = {\n  max_rows = 500; # max logs in one bulk req to elastic and first reason to flush buffer to elastic\n  max_interval = 60; # seconds; if the first log in the buffer is older than this interval, flush the buffer\n  max_fail = 15; # max consecutive failures before giving up\n};\nindex_template = {\n  managed = true;\n  name = "rspamd";\n  priority = 0;\n  pattern = "{service}-%Y.%m.%d"; # {service} is replaced with index_template.name\n  shards_count = 3;\n  replicas_count = 1;\n  refresh_interval = 5; # seconds\n  dynamic_keyword_ignore_above = 256;\n  headers_count_ignore_above = 5; # record only N first same named headers, add "ignored above..." if reached, set 0 to disable limit\n  headers_text_ignore_above = 2048; # strip specific header value and add "..." to the end; set 0 to disable limit\n  symbols_nested = false;\n  urls_nested = false;\n  empty_value = "unknown"; # empty numbers, ips and ipnets are not customizable they will be always 0, :: and ::/128 respectively\n};\nindex_policy = {\n  enabled = true;\n  managed = true;\n  name = "rspamd"; # if you want use custom lifecycle policy, change name and set managed = false\n  hot = {\n    index_priority = 100;\n  };\n  warm = {\n    enabled = true;\n    after = "2d";\n    index_priority = 50;\n    migrate = true; # only supported with elastic distro, will not have impact elsewhere\n    read_only = true;\n    change_replicas = false;\n    replicas_count = 1;\n    shrink = false;\n    shards_count = 1;\n    max_gb_per_shard = 0; # zero - disabled by default, if enabled - shards_count is ignored\n    force_merge = false;\n    segments_count = 1;\n  };\n  cold = {\n    enabled = true;\n    after = "14d";\n    index_priority = 0;\n    migrate = true; # only supported with elastic distro, will not have impact elsewhere\n    read_only = true;\n    change_replicas = false;\n    replicas_count = 1;\n  };\n  delete = {\n    enabled = true;\n    after = "30d";\n  };\n};\n# Base set of collected headers (overridable):\ncollect_headers = ["From", "To", "Subject", "Date", "User-Agent"];\n# Add extra headers on top of the base list:\nextra_collect_headers = [\n  # "Precedence",\n  # "List-Id",\n  # "X-Mailer",\n];\ncollect_urls = {\n  enabled = true;\n  max_urls = 0; # 0 = no limit; truncation keeps CTAs and most-repeated URLs first\n  separate_cta = true; # emit CTA URLs into a separate urls_cta field\n  full_urls = false; # log full URL text instead of just the host\n  full_cta_urls = false; # same, for CTA URLs\n  # params passed to task:get_urls()\n  get_url_params = {\n    content = true; # URLs from text/html content\n    images = false; # URLs from <img src> attributes\n    emails = false; # mailto: URLs\n    protocols = ["http", "https", "ftp"];\n  };\n};\ngeoip = {\n  enabled = true;\n  managed = true;\n  pipeline_name = "rspamd-geoip";\n};\n'})})]})}function u(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},28453:(e,n,t)=>{t.d(n,{R:()=>i,x:()=>o});var s=t(96540);const a={},r=s.createContext(a);function i(e){const n=s.useContext(r);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:i(e.components),s.createElement(r.Provider,{value:n},e.children)}}}]);