1"use strict";(self.webpackChunkrspamd_docs=self.webpackChunkrspamd_docs||[]).push([[5557],{28453:(e,n,s)=>{s.d(n,{R:()=>i,x:()=>a});var t=s(96540);const r={},d=t.createContext(r);function i(e){const n=t.useContext(d);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:i(e.components),t.createElement(d.Provider,{value:n},e.children)}},73347:(e,n,s)=>{s.r(n),s.d(n,{assets:()=>c,contentTitle:()=>a,default:()=>h,frontMatter:()=>i,metadata:()=>t,toc:()=>l});const t=JSON.parse('{"id":"modules/metadata_exporter","title":"Metadata exporter","description":"The Metadata exporter operates on a set of rules that identify interesting messages, and subsequently sends information based on these rules to an external service. The exporter supports Redis Pub/Sub, HTTP POST, and SMTP as built-in backends, while also allowing users to define custom backends as desired.","source":"@site/docs/modules/metadata_exporter.md","sourceDirName":"modules","slug":"/modules/metadata_exporter","permalink":"/modules/metadata_exporter","draft":false,"unlisted":false,"editUrl":"https://github.com/rspamd/docs.rspamd.com/edit/master/docs/modules/metadata_exporter.md","tags":[],"version":"current","frontMatter":{"title":"Metadata exporter"},"sidebar":"docs","previous":{"title":"Mailing lists module","permalink":"/modules/maillist"},"next":{"title":"Metric exporter","permalink":"/modules/metric_exporter"}}');var r=s(74848),d=s(28453);const i={title:"Metadata exporter"},a="Metadata exporter",c={},l=[{value:"Theory of operation",id:"theory-of-operation",level:3},{value:"Configuration",id:"configuration",level:3},{value:"Stock pushers (backends)",id:"stock-pushers-backends",level:3},{value:"Stock selectors",id:"stock-selectors",level:3},{value:"Stock formatters",id:"stock-formatters",level:3},{value:"Settings: general",id:"settings-general",level:3},{value:"Settings: <code>http</code> backend",id:"settings-http-backend",level:3},{value:"Settings: <code>redis_pubsub</code> backend",id:"settings-redis_pubsub-backend",level:3},{value:"Settings: <code>redis_stream</code> backend",id:"settings-redis_stream-backend",level:3},{value:"Settings: <code>json_raw_tcp</code> backend",id:"settings-json_raw_tcp-backend",level:3},{value:"Settings: <code>send_mail</code> backend",id:"settings-send_mail-backend",level:3},{value:"General metadata",id:"general-metadata",level:3},{value:"Custom functions",id:"custom-functions",level:3},{value:"Examples",id:"examples",level:3},{value:"Python Receiver for <code>multipart</code> formatter",id:"python-receiver-for-multipart-formatter",level:4},{value:"Structured formatter (4.0+)",id:"structured-formatter-40",level:3},{value:"Structured formatter output fields",id:"structured-formatter-output-fields",level:4},{value:"Structured formatter features",id:"structured-formatter-features",level:4},{value:"Zstd compression option",id:"zstd-compression-option",level:4}];function o(e){const n={a:"a",b:"b",code:"code",em:"em",h1:"h1",h3:"h3",h4:"h4",header:"header",li:"li",mark:"mark",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,d.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(n.header,{children:(0,r.jsx)(n.h1,{id:"metadata-exporter",children:"Metadata exporter"})}),"\n",(0,r.jsx)(n.p,{children:"The Metadata exporter operates on a set of rules that identify interesting messages, and subsequently sends information based on these rules to an external service. The exporter supports Redis Pub/Sub, HTTP POST, and SMTP as built-in backends, while also allowing users to define custom backends as desired."}),"\n",(0,r.jsx)(n.p,{children:"Potential applications of the Metadata exporter include quarantining, logging, alerting, and feedback loops."}),"\n",(0,r.jsx)(n.h3,{id:"theory-of-operation",children:"Theory of operation"}),"\n",(0,r.jsx)(n.p,{children:"For each rule defined in configuration:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["A ",(0,r.jsx)(n.code,{children:"selector"})," function identifies messages that we want to export metadata from (default selector selects all messages)."]}),"\n",(0,r.jsxs)(n.li,{children:["A ",(0,r.jsx)(n.code,{children:"formatter"})," function extracts formatted metadata from the message (default formatter returns full message content)."]}),"\n",(0,r.jsxs)(n.li,{children:["A ",(0,r.jsx)(n.code,{children:"pusher"})," function (defined by the ",(0,r.jsx)(n.code,{children:"backend"})," setting) pushes the formatted metadata somewhere"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"A number of such functions are defined in the plugin which can be used in addition to user-defined functions."}),"\n",(0,r.jsx)(n.h3,{id:"configuration",children:"Configuration"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-hcl",children:'metadata_exporter {\n\n # Each rule defines some export process\n\n rules {\n\n # The following rule posts JSON-formatted metadata at the defined URL\n # when it sees a rejected mail from an authenticated user\n MY_HTTP_ALERT_1 {\n backend = "http";\n url = "http://127.0.0.1:8080/foo";\n # More about selectors and formatters later\n selector = "is_reject_authed";\n formatter = "json";\n }\n\n # This rule posts all messages to a Redis Pub/Sub channel\n MY_REDIS_PUBSUB_1 {\n backend = "redis_pubsub";\n channel = "foo";\n # Default formatter and selector is used\n }\n\n # This rule sends an e-Mail alert over SMTP containing message metadata\n # when it sees a rejected mail from an authenticated user\n MY_EMAIL_1 {\n backend = "send_mail";\n smtp = "127.0.0.1";\n mail_to = "[email protected]";\n selector = "is_reject_authed";\n formatter = "email_alert";\n }\n\n }\n\n}\n'})}),"\n",(0,r.jsx)(n.h3,{id:"stock-pushers-backends",children:"Stock pushers (backends)"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"http"}
1),": sends content over HTTP POST"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"json_raw_tcp"}),": sends JSON content over a raw TCP connection"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"redis_pubsub"}),": sends content over Redis Pub/Sub"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"redis_stream"})," (4.0+): sends content to a Redis Stream"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"send_mail"}),": sends content over SMTP"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"stock-selectors",children:"Stock selectors"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"default"}),": selects all mail"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"is_spam"}),": matches messages with ",(0,r.jsx)(n.code,{children:"reject"}),", ",(0,r.jsx)(n.code,{children:"add header"}),", or ",(0,r.jsx)(n.code,{children:"rewrite subject"})," action"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"is_spam_authed"}),": matches messages with ",(0,r.jsx)(n.code,{children:"reject"}),", ",(0,r.jsx)(n.code,{children:"add header"}),", or ",(0,r.jsx)(n.code,{children:"rewrite subject"})," action from authenticated users"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"is_reject"}),": matches messages with ",(0,r.jsx)(n.code,{children:"reject"})," action"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"is_reject_authed"}),": matches messages with ",(0,r.jsx)(n.code,{children:"reject"})," action from authenticated users"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"is_not_soft_reject"}),": matches all messages except those with ",(0,r.jsx)(n.code,{children:"soft reject"})," action"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"stock-formatters",children:"Stock formatters"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"default"}),": returns full message content"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"email_alert"}),": generates an e-Mail report about the message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"json"}),": returns JSON-formatted metadata about a message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"multipart"})," (3.14.2+): Sends metadata as JSON part + raw message as ",(0,r.jsx)(n.code,{children:"message/rfc822"})," part using standard ",(0,r.jsx)(n.code,{children:"multipart/form-data"})]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"msgpack"})," (3.14.2+): Binary MessagePack format with embedded message (efficient for binary data)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"json_with_message"})," (3.14.2+): JSON with base64-encoded message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"structured"})," (4.0+): Rich structured export with Rspamd UUID correlation, extracted text, attachments, images, URLs in MessagePack format"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"settings-general",children:"Settings: general"}),"\n",(0,r.jsx)(n.p,{children:"The following settings can be defined on any rule:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"selector"}),": defines selector for the rule"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"formatter"}),": defines formatter for the rule"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"backend"}),": defines backend (pusher) for the rule"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"defer"}),": if true, ",(0,r.jsx)(n.code,{children:"soft reject"})," action is forced on failed processing"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"timeout"}),": defines module timeout (default: '5s')"]}),"\n"]}),"\n",(0,r.jsxs)(n.h3,{id:"settings-http-backend",children:["Settings: ",(0,r.jsx)(n.code,{children:"http"})," backend"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"url"})," (required): defines the URL to post content to"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"meta_header_prefix"}),": prefix for meta headers (default: ",(0,r.jsx)(n.code,{children:"'X-Rspamd-'"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"meta_headers"})," (bool): if set to ",(0,r.jsx)(n.code,{children:"true"}),", general metadata is added to HTTP request headers (default: ",(0,r.jsx)(n.code,{children:"false"}
1),"). ",(0,r.jsx)(n.strong,{children:"Deprecated in 3.14.2"}),": Use ",(0,r.jsx)(n.code,{children:'formatter = "multipart"'})," or ",(0,r.jsx)(n.code,{children:'formatter = "msgpack"'})," instead."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"mime_type"}),": defines the MIME type of the content sent in the HTTP POST"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"user"})," & ",(0,r.jsx)(n.code,{children:"password"}),": if both parameters are set, Basic authentication will be used"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"gzip"})," (bool): specifies whether the payload needs to be sent with gzip compression (default: ",(0,r.jsx)(n.code,{children:"false"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"keepalive"})," (bool): specifies whether the connection should use keepalive (default: ",(0,r.jsx)(n.code,{children:"false"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"no_ssl_verify"})," (bool): disable SSL certificate verification (default: ",(0,r.jsx)(n.code,{children:"false"}),")"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"connect_timeout"}),": timeout for establishing the TCP connection"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"ssl_timeout"}),": timeout for SSL handshake"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"write_timeout"}),": timeout for writing the request body"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"read_timeout"}),": timeout for reading the response"]}),"\n"]}),"\n",(0,r.jsxs)(n.h3,{id:"settings-redis_pubsub-backend",children:["Settings: ",(0,r.jsx)(n.code,{children:"redis_pubsub"})," backend"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"channel"})," (required): defines Pub/Sub channel to post content to"]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["See ",(0,r.jsx)(n.a,{href:"/configuration/redis",children:"here"})," for information on configuring Redis servers."]}),"\n",(0,r.jsxs)(n.h3,{id:"settings-redis_stream-backend",children:["Settings: ",(0,r.jsx)(n.code,{children:"redis_stream"})," backend"]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.em,{children:"Available since version 4.0"})}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"stream_key"})," (required): defines Redis Stream key to append content to"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"max_len"}),": optional maximum length for the stream (uses ",(0,r.jsx)(n.code,{children:"MAXLEN ~"})," for approximate trimming)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"per_recipient"}),": if ",(0,r.jsx)(n.code,{children:"true"}),", creates per-recipient streams by appending ",(0,r.jsx)(n.code,{children:":recipient@address"})," to ",(0,r.jsx)(n.code,{children:"stream_key"})]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["The backend uses Redis ",(0,r.jsx)(n.code,{children:"XADD"})," command. This is useful for building event-driven pipelines with consumer groups."]}),"\n",(0,r.jsxs)(n.h3,{id:"settings-json_raw_tcp-backend",children:["Settings: ",(0,r.jsx)(n.code,{children:"json_raw_tcp"})," backend"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"host"})," (required): hostname or IP address of the TCP server"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"port"})," (required): TCP port to connect to"]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["The backend sends the formatted content as-is over a raw TCP connection without an application-layer framing protocol. No response is read from the server. Combine with the ",(0,r.jsx)(n.code,{children:"json"})," formatter to push newline-delimited JSON to a log aggregator or SIEM."]}),"\n",(0,r.jsxs)(n.h3,{id:"settings-send_mail-backend",children:["Settings: ",(0,r.jsx)(n.code,{children:"send_mail"})," backend"]}),"\n",(0,r.jsxs)(n.p,{children:["If the ",(0,r.jsx)(n.code,{children:"send_mail"})," backend is used with the default formatter, the original spam message content will be analyzed by Rspamd and is highly likely matched as spam."]}),"\n",(0,r.jsxs)(n.p,{children:["When ",(0,r.jsx)(n.code,{children:"send_mail"})," backend is used in conjunction with ",(0,r.jsx)(n.code,{children:"email_alert"}
1)," formatter, the URLs found in the symbols options will be analysed by Rspamd and the report will be matched as spam possibly."]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsxs)(n.mark,{children:["To prevent ",(0,r.jsx)(n.b,{children:"looping"}),", it is essential to ensure that email messages from the Metadata exporter are ",(0,r.jsx)(n.b,{children:"not scanned"})," by Rspamd."]})," This can be achieved by setting up a specific Postfix Transport to bypass Rspamd, or by allowing the recipient of the ",(0,r.jsx)(n.code,{children:"email_alert"})," to receive spam."]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"smtp"})," (required): hostname of SMTP server"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"mail_to"})," (required): recipient of e-mail alert"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"mail_from"}),": Sender address (default empty)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"email_alert_user"})," (1.7.0+, default false): Send a copy of the alert to the authenticated SMTP username"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"email_alert_sender"})," (1.7.0+, default false): Send a copy of the alert to the SMTP sender (NB: please ensure that it can be trusted)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"email_alert_recipients"})," (1.7.0+, default false): Send a copy of the alert to SMTP recipients (NB: please ensure they can be trusted; don't use this?)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"email_template"}),": template used for alert (default shown below)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"helo"}),": HELO to send (default 'rspamd')"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"smtp_port"}),": SMTP port if not 25"]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["The default value for ",(0,r.jsx)(n.code,{children:"email_template"})," is as follows:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{children:'From: "Rspamd" <$mail_from>\nTo: $mail_to\nSubject: Spam alert\nDate: $date\nMIME-Version: 1.0\nMessage-ID: <$our_message_id>\nContent-type: text/plain; charset=utf-8\nContent-Transfer-Encoding: 8bit\n\nAuthenticated username: $user\nIP: $ip\nQueue ID: $qid\nSMTP FROM: $from\nSMTP RCPT: $rcpt\nMIME From: $header_from\nMIME To: $header_to\nMIME Date: $header_date\nSubject: $header_subject\nMessage-ID: $message_id\nAction: $action\nScore: $score\nSymbols: $symbols\n'})}),"\n",(0,r.jsx)(n.p,{children:"Variables can be substituted according to general metadata keys described in the next section."}),"\n",(0,r.jsx)(n.h3,{id:"general-metadata",children:"General metadata"}),"\n",(0,r.jsxs)(n.p,{children:["Metadata as returned by the ",(0,r.jsx)(n.code,{children:"json"})," formatter can be referenced by key in ",(0,r.jsx)(n.code,{children:"email_template"}),". The following keys are defined:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"action"}),": metric action for message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"from"}),": SMTP FROM"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"header_date"}),": Contents of Date header(s)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"header_from"}),": Contents of From header(s)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"header_subject"}),": Contents of Subject header(s)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"header_to"}),": Contents of To header(s)"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"ip"}),": IP of message sender"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"mail_from"})," (",(0,r.jsx)(n.code,{children:"email_template"})," only): sender of alert"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"mail_to"})," (",(0,r.jsx)(n.code,{children:"email_template"})," only): recipient of alert"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"message_id"}),": Message-ID of original message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"our_message_id"})," (",(0,r.jsx)(n.code,{children:"email_template"})," only): message-ID generated for alert"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"qid"}),": Queue-ID of message provided by MTA"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"rcpt"}),": SMTP RCPT"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"score"}),": Metric score of the message"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"symbols"}),": Symbols in metric"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"user"}),": authenticated username of message sender"]}),"\n"]}),"\n",(0,r.jsx)(n.h3,{id:"custom-functions",children:"Custom functions"}),"\n",(0,r.jsxs)(n.p,{children:["It is possible to define custom selectors/pushers/backends. Functions are defined in the ",(0,r.jsx)(n.code,{children:"custom_select"}),"/",(0,r.jsx)(n.code,{children:"custom_format"}),"/",(0,r.jsx)(n.code,{children:"custom_push"})," groups and referenced by name in the ",(0,r.jsx)(n.code,{children:"selector"}),"/",(0,r.jsx)(n.code,{children:"formatter"}),"/",(0,r.jsx)(n.code,{children:"backend"})," settings:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-hcl",children:'metadata_exporter {\n\n # Define custom selector(s)\n custom_select {\n mine = <<EOD\nreturn function(task)\n -- Select all messages\n return true\nend\nEOD;\n }\n\n # Define custom formatter(s)\n custom_format {\n mine = <<EOD\nreturn function(task)\n -- Push message ID\n return task:get_message_id()\nend\nEOD;\n }\n\n # Define custom backend(s)\n custom_push {\n mine = <<EOD\nreturn function (task, data, rule)\n -- Log payload\n local rspamd_logger = require "rspamd_logger"\n rspamd_logger.infox(task, \'METATEST %s\', data)\nend\nEOD;\n }\n\n rules {\n\n CUSTOM_EXPORT {\n selector = "mine";\n formatter = "mine";\n backend = "mine";\n }\n\n }\n\n}\n'})}),"\n",(0,r.jsx)(n.h3,{id:"examples",children:"Examples"}),"\n",(0,r.jsxs)(n.h4,{id:"python-receiver-for-multipart-formatter",children:["Python Receiver for ",(0,r.jsx)(n.code,{children:"multipart"})," formatter"]}),"\n",(0,r.jsxs)(n.p,{children:["This example demonstrates how to build a simple service using ",(0,r.jsx)(n.code,{children:"aiohttp"})," that accepts metadata and messages sent by the ",(0,r.jsx)(n.code,{children:"metadata_exporter"})," with ",(0,r.jsx)(n.code,{children:'formatter = "multipart"'}),"."]}),"\n",(0,r.jsx)(n.p,{children:"It conceptually shows how to quarantine messages to Kafka or Cassandra."}),"\n",(0,r.jsx)(n.p,{children:"Prerequisites:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"pip install aiohttp aiokafka cassandra-driver\n"})}),"\n",(0,r.jsx)(n.p,{children:"receiver.py:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-python",children:'import asyncio\nimport json\nimport logging\nfrom aiohttp import web\n# Optional: import for Kafka/Cassandra\n# from aiokafka import AIOKafkaProducer\n# from cassandra.cluster import Cluster\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger("rspamd-receiver")\n\nasync def handle_push(request):\n """\n Handle POST request from Rspamd metadata_exporter (multipart).\n Expected parts:\n - \'metadata\': JSON object with Rspamd metadata\n - \'message\': Raw message content (optional)\n """\n reader = await request.multipart()\n metadata = {}\n message_content = b""\n\n # Iterate through multipart fields\n async for field in reader:\n if field.name == \'metadata\':\n # Parse metadata JSON\n try:\n raw_json = await field.read()\n metadata = json.loads(raw_json)\n logger.info(f"Received metadata for Message-ID: {metadata.get(\'message_id\')}")\n except Exception as e:\n logger.error(f"Failed to parse metadata: {e}")\n return web.Response(status=400, text="Invalid Metadata")\n \n elif field.name == \'message\':\n # Read raw message content\n message_content = await field.read()\n logger.info(f"Received message content ({len(message_content)} bytes)")\n \n if not metadata:\n return web.Response(status=400, text="Missing Metadata")\n\n # --- Quarantine Logic Ex
1ample ---\n \n # 1. Kafka Example (Async)\n # await produce_to_kafka(metadata, message_content)\n \n # 2. Cassandra Example (Async)\n # await save_to_cassandra(metadata, message_content)\n \n logger.info("Message processed successfully")\n return web.Response(text="OK")\n\n# Mock functions for illustration\nasync def produce_to_kafka(metadata, content):\n # producer = AIOKafkaProducer(bootstrap_servers=\'localhost:9092\')\n # await producer.start()\n # try:\n # await producer.send_and_wait("rspamd-quarantine", value=content, key=metadata.get(\'message_id\').encode())\n # finally:\n # await producer.stop()\n pass\n\nasync def save_to_cassandra(metadata, content):\n # loop = asyncio.get_event_loop()\n # cluster = Cluster([\'127.0.0.1\'])\n # session = cluster.connect(\'mail_quarantine\')\n # stmt = "INSERT INTO messages (id, metadata, content) VALUES (%s, %s, %s)"\n # await loop.run_in_executor(None, session.execute, stmt, (metadata[\'message_id\'], json.dumps(metadata), content))\n pass\n\napp = web.Application()\napp.add_routes([web.post(\'/push\', handle_push)])\n\nif __name__ == \'__main__\':\n web.run_app(app, port=8080)\n'})}),"\n",(0,r.jsx)(n.p,{children:"Configure Rspamd to use this receiver:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-hcl",children:'metadata_exporter {\n rules {\n QUARANTINE {\n backend = "http";\n url = "http://127.0.0.1:8080/push";\n selector = "is_reject"; # Export rejected messages\n formatter = "multipart"; # Send metadata + raw message\n }\n }\n}\n'})}),"\n",(0,r.jsx)(n.h3,{id:"structured-formatter-40",children:"Structured formatter (4.0+)"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"structured"})," formatter provides rich, analysis-ready metadata in MessagePack format with Rspamd UUID correlation:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-hcl",children:'metadata_exporter {\n rules {\n STRUCTURED_EXPORT {\n backend = "redis_stream";\n formatter = "structured";\n stream_key = "rspamd:events";\n max_len = 10000;\n # Optional: compress text/attachments with zstd\n zstd_compress = true;\n }\n }\n}\n'})}),"\n",(0,r.jsx)(n.h4,{id:"structured-formatter-output-fields",children:"Structured formatter output fields"}),"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Field"}),(0,r.jsx)(n.th,{children:"Type"}),(0,r.jsx)(n.th,{children:"Description"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"uuid"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsxs)(n.td,{children:["Rspamd internal message UUID (from ",(0,r.jsx)(n.code,{children:"task:get_uuid()"}),")"]})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"ip"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsx)(n.td,{children:"Sender IP address"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"from"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsx)(n.td,{children:"SMTP envelope sender"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"rcpt"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsx)(n.td,{children:"SMTP envelope recipient"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"user"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsx)(n.td,{children:"Authenticated username"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"score"})}),(0,r.jsx)(n.td,{children:"Number"}),(0,r.jsx)(n.td,{children:"Spam score"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"action"})}),(0,r.jsx)(n.td,{children:"String"}),(0,r.jsx)(n.td,{children:"Rspamd action"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"symbols"})}),(0,r.jsx)(n.td,{children:"Object"}),(0,r.jsx)(n.td,{children:"Symbol results"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"text"})}),(0,r.jsx)(n.td,{children:"String/Binary"}),(0,r.jsx)(n.td,{children:"Extracted plain text (optionally zstd-compressed)"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"text_truncated"})}),(0,r.jsx)(n.td,{children:"Boolean"}),(0,r.jsx)(n.td,{children:"True if text was truncated (max 32KB)"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"text_compressed"})}),(0,r.jsx)(n.td,{children:"Boolean"}),(0,r.jsx)(n.td,{children:"True if text is zstd-compressed"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"attachments"})}),(0,r.jsx)(n.td,{children:"Array"}),(0,r.jsx)(n.td,{children:"Attachment metadata with content"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"images"})}),(0,r.jsx)(n.td,{children:"Array"}),(0,r.jsx)(n.td,{children:"Embedded image metadata with content"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"urls"})}),(0,r.jsx)(n.td,{children:"Array"}),(0,r.jsx)(n.td,{children:"Extracted URLs with host/TLD"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"is_reply"})}),(0,r.jsx)(n.td,{children:"Boolean"}),(0,r.jsx)(n.td,{children:"True if message has In-Reply-To header"})]})]})]}),"\n",(0,r.jsx)(n.h4,{id:"structured-formatter-features",children:"Structured formatter features"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"UUID correlation"}),": Rspamd's internal message UUID enables cross-system correlation via the injected ",(0,r.jsx)(n.code,{children:"X-Rspamd-UUID"})," header"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"X-Rspamd-UUID header"}
1),": Automatically injected into message for IMAP/external correlation"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Smart text extraction"}),": Cleaned, reply-trimmed text up to 32KB"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Attachment analysis"}),": Includes detected MIME type (not just announced), size, digest, and optional content"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"URL extraction"}),": Up to 100 URLs with host and TLD information"]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Zstd compression"}),": Optional compression for text, attachments, and images to reduce storage"]}),"\n"]}),"\n",(0,r.jsx)(n.h4,{id:"zstd-compression-option",children:"Zstd compression option"}),"\n",(0,r.jsxs)(n.p,{children:["When ",(0,r.jsx)(n.code,{children:"zstd_compress = true"})," is set:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Text, attachment content, and image content are compressed with zstd"}),"\n",(0,r.jsxs)(n.li,{children:["Compressed fields include ",(0,r.jsx)(n.code,{children:"content_compressed = true"})," or ",(0,r.jsx)(n.code,{children:"text_compressed = true"})]}),"\n",(0,r.jsx)(n.li,{children:"Consumer must decompress using zstd library"}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,d.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(o,{...e})}):o(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.