https://www.rabbitmq.com/assets/js/fcf5edbe.7d579788.js

1"use strict";(self.webpackChunkrabbitmq_website=self.webpackChunkrabbitmq_website||[]).push([["15370"],{45514(e,n,s){s.r(n),s.d(n,{metadata:()=>i,default:()=>u,frontMatter:()=>a,contentTitle:()=>o,toc:()=>d,assets:()=>l});var i=JSON.parse('{"id":"persistence-conf","title":"Persistence Configuration","description":"\x3c!--","source":"@site/versioned_docs/version-4.1/persistence-conf.md","sourceDirName":".","slug":"/persistence-conf","permalink":"/docs/4.1/persistence-conf","draft":false,"unlisted":false,"editUrl":"https://github.com/rabbitmq/rabbitmq-website/tree/main/versioned_docs/version-4.1/persistence-conf.md","tags":[],"version":"4.1","frontMatter":{"title":"Persistence Configuration"},"sidebar":"docsSidebar","previous":{"title":"Runtime Tuning","permalink":"/docs/4.1/runtime"},"next":{"title":"Deployment Guidelines","permalink":"/docs/4.1/production-checklist"}}'),t=s(74848),r=s(28453);let a={title:"Persistence Configuration"},o="Persistence Configuration",l={},d=[{value:"Overview",id:"overview",level:2},{value:"Overview of Persistence in RabbitMQ",id:"overview",level:2},{value:"Streams",id:"streams",level:2},{value:"Quorum Queues",id:"quorum-queues",level:2},{value:"Classic Queues",id:"classic-queues",level:2},{value:"Queue Version",id:"queue-version",level:3},{value:"How Classic Queue v1 Persistence Overview",id:"cq-v1",level:2},{value:"Memory Costs",id:"memory-costs",level:3},{value:"Message Embedding in Queue Indices",id:"index-embedding",level:3},{value:"OS and Runtime Limits Affecting",id:"limits",level:2},{value:"Too Few File Handles",id:"file-handles",level:3},{value:"Classic Queues v1: Alternate Message Store Index Implementations",id:"msg-store-index-implementations",level:2}];function h(e){let n={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(n.header,{children:(0,t.jsx)(n.h1,{id:"persistence-configuration",children:"Persistence Configuration"})}),"\n",(0,t.jsx)(n.h2,{id:"overview",children:"Overview"}),"\n",(0,t.jsxs)(n.p,{children:["This guide covers a few configurable\nvalues that affect throughput, latency and I/O characteristics of a node.\nConsider reading the entire guide and get accustomed to ",(0,t.jsx)(n.a,{href:"https://rabbitmq.github.io/rabbitmq-perf-test/stable/htmlsingle/",children:"benchmarking with PerfTest"}),"\nbefore drawing any conclusions."]}),"\n",(0,t.jsx)(n.p,{children:"Some related guides include:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.a,{href:"./configure",children:"Main configuration guide"})}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.a,{href:"./relocate",children:"File and Directory Locations"})}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.a,{href:"./runtime",children:"Runtime Tuning"})}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"./queues#runtime-characteristics",children:"Queues"})," and their runtime characteristics"]}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.a,{href:"./quorum-queues",children:"Quorum Queues"})}),"\n",(0,t.jsx)(n.li,{children:(0,t.jsx)(n.a,{href:"./streams",children:"Streams"})}),"\n"]}),"\n",(0,t.jsx)(n.h2,{id:"overview",children:"Overview of Persistence in RabbitMQ"}),"\n",(0,t.jsx)(n.p,{children:"Modern RabbitMQ versions provide several queue types plus streams:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"./quorum-queues",children:"Quorum queues"}),": replicated, durable, data-safety oriented"]}),"\n",(0,t.jsxs)(n.li,{children:[(0,t.jsx)(n.a,{href:"./streams",children:"Streams"}),": a replicated, durable data structure that supports different operations (than a queue)"]}),"\n",(0,t.jsx)(n.li,{children:"Classic queues: the original queue type, single replica only starting with RabbitMQ 4.0"}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"These queue types have different storage implementations and applicable configuration\nsettings that can be tuned are also different."}),"\n",(0,t.jsx)(n.h2,{id:"streams",children:"Streams"}),"\n",(0,t.jsxs)(n.p,{children:["Streams use a log-based storage mechanism and keep very little data in memory\n(primarily the operational data that has not yet been written to disk).\nNonetheless they offer excellent throughput when clients use the ",(0,t.jsx)(n.a,{href:"./stream",children:"RabbitMQ Stream Protocol"}),"."]}),"\n",(0,t.jsx)(n.p,{children:"Since streams are very disk I/O heavy, their throughput degrades with larger messages.\nThey benefit greatly from modern SSD and NVMe storage."}),"\n",(0,t.jsx)(n.p,{children:"Streams offer no tunable storage parameters related to storage."}),"\n",(0,t.jsx)(n.h2,{id:"quorum-queues",children:"Quorum Queues"}),"\n",(0,t.jsx)(n.p,{children:"Quorum queues use a log-based storage mechanism implemented by RabbitMQ's Raft\nimplementation. They keep very little data in memory\n(primarily the operational data that has not yet been written to disk)."}),"\n",(0,t.jsx)(n.p,{children:"As quorum queues persist all data to disks before doing anything it is recommended\nto use the fastest disks possible."}),"\n",(0,t.jsx)(n.p,{children:"Due to the disk I/O-heavy nature of quorum queues, their throughput decreases\nas message sizes increase."}),"\n",(0,t.jsx)(n.p,{children:"The primary storage-related setting that can affect quorum queue resource use\nis the write-ahead log segment size limit, the limit at which WAL in-memory\ntable will be moved to disk. In other words, every quorum queue would be able to\nkeep up to this much message data in memory under steady load."}),"\n",(0,t.jsx)(n.p,{children:"The limit can be controlled"}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ini",children:"# Flush current WAL file to a segment file on disk once it reaches 32 MiB in size\nraft.wal_max_size_bytes = 32000000\n"})}),"\n",(0,t.jsx)(n.admonition,{type:"important",children:(0,t.jsxs)(n.p,{children:["Because memory is not guaranteed to be deallocated instantly by the ",(0,t.jsx)(n.a,{href:"./runtime/",children:"runtime"}),",\nwe recommend that the RabbitMQ node is allocated at least 3 times the memory of the effective WAL file size limit.\nMore will be required in high-throughput systems.\

1n4 times is a good starting point for those."]})}),"\n",(0,t.jsx)(n.h2,{id:"classic-queues",children:"Classic Queues"}),"\n",(0,t.jsx)(n.p,{children:"Classic queues have two storage implementations available to them: v1 (the original\none) and v2 (available in RabbitMQ 3.10 and later versions)."}),"\n",(0,t.jsx)(n.h3,{id:"queue-version",children:"Queue Version"}),"\n",(0,t.jsxs)(n.p,{children:["Since ",(0,t.jsx)(n.strong,{children:"RabbitMQ 3.10.0"}),", the broker has a new implementation of\nclassic queues, named ",(0,t.jsx)(n.strong,{children:"version 2"}),". Version 2 queues have a new\nindex file format and implementation as well as a new per-queue\nstorage file format to replace the embedding of messages directly\nin the index."]}),"\n",(0,t.jsx)(n.p,{children:"The main improvement from version 2 is improved stability while\nunder high memory pressure."}),"\n",(0,t.jsxs)(n.p,{children:["In ",(0,t.jsx)(n.strong,{children:"RabbitMQ 3.10.0"})," version 1 remains the default. It is possible\nto switch back and forth between version 1 and version 2."]}),"\n",(0,t.jsxs)(n.p,{children:["The version can be changed using the ",(0,t.jsx)(n.code,{children:"queue-version"})," ",(0,t.jsx)(n.a,{href:"./policies",children:"policy"})," key.\nWhen setting a new version via policy the queue will immediately\nconvert its data on disk. It is possible to upgrade to version 2\nor downgrade to version 1. Note that for large queues the conversion\nmay take some time and results in the queue being unavailable while\nthe conversion is running."]}),"\n",(0,t.jsxs)(n.p,{children:["The default version can be set through configuration by setting\n",(0,t.jsx)(n.code,{children:"classic_queue.default_version"})," in ",(0,t.jsx)(n.code,{children:"rabbitmq.conf"}),":"]}),"\n",(0,t.jsx)(n.pre,{children:(0,t.jsx)(n.code,{className:"language-ini",children:"# makes classic queues use a more efficient message storage\n# and queue index implementations\nclassic_queue.default_version = 2\n"})}),"\n",(0,t.jsx)(n.h2,{id:"cq-v1",children:"How Classic Queue v1 Persistence Overview"}),"\n",(0,t.jsx)(n.p,{children:'First, some background: both persistent and transient messages\ncan be written to disk. Persistent messages will be written to\ndisk as soon as they reach the queue, while transient messages\nwill be written to disk only so that they can be evicted from\nmemory while under memory pressure. Persistent messages are also\nkept in memory when possible and only evicted from memory under\nmemory pressure. The "persistence layer" refers to the mechanism\nused to store messages of both types to disk.'}),"\n",(0,t.jsx)(n.p,{children:'On this page we say "queue" to refer to a non-replicated queue or a\nqueue leader or a queue mirror. Queue mirroring is a "layer above"\npersistence.'}),"\n",(0,t.jsxs)(n.p,{children:["The persistence layer has two components: the ",(0,t.jsx)(n.em,{children:"queue index"}),"\nand the ",(0,t.jsx)(n.em,{children:"message store"}),". The queue index is responsible for\nmaintaining knowledge about where a given message is in a queue,\nalong with whether it has been delivered and acknowledged. There\nis therefore one queue index per queue."]}),"\n",(0,t.jsx)(n.p,{children:'The message store is a key-value store for messages, shared\namong all queues in each vhost. Messages (the body, and any\nmetadata fields: properties and/or headers) can either be stored\ndirectly in the queue index, or written to the message store. There are\ntechnically two message stores (one for transient and one for\npersistent messages) but they are usually considered together as\n"the message store".'}),"\n",(0,t.jsx)(n.h3,{id:"memory-costs",children:"Memory Costs"}),"\n",(0,t.jsx)(n.p,{children:"Under memory pressure, the persistence layer tries to write as\nmuch out to disk as possible, and remove as much as possible\nfrom memory. There are some things however which must remain in\nmemory:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:["Each queue maintains some metadata for each\n",(0,t.jsx)(n.em,{children:"unacknowledged"})," message. The message itself can be\nremoved from memory if its destination is the message store."]}),"\n",(0,t.jsx)(n.li,{children:"The message store needs an index. The default message store\nindex uses a small amount of memory for every message in the\nstore."}),"\n"]}),"\n",(0,t.jsx)(n.h3,{id:"index-embedding",children:"Message Embedding in Queue Indices"}),"\n",(0,t.jsx)(n.p,{children:"There are advantages and disadvantages to writing messages to\nthe queue index."}),"\n",(0,t.jsx)(n.p,{children:"This feature has advantages and disadvantages. Main advantages are:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"Messages can be written to disk in one operation rather than\ntwo; for tiny messages this can be a substantial gain."}),"\n",(0,t.jsx)(n.li,{children:"Messages that are written to the queue index do not require an\nentry in the message store index and thus do not have a memory\ncost when paged out."}),"\n"]}),"\n",(0,t.jsx)(n.p,{children:"Disadvantages are:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsx)(n.li,{children:"The queue index keeps blocks of a fixed number of records in\nmemory; if non-tiny messages are written to the queue index then\nmemory use can be substantial."}),"\n",(0,t.jsx)(n.li,{children:"If a message is routed to multiple queues by an exchange, the\nmessage will need to be written to multiple queue indices. If\nsuch a message is written to the message store, only one copy\nneeds to be written."}),"\n",(0,t.jsx)(n.li,{children:"Unacknowledged messages whose destination is the queue index\nare always kept in memory."}),"\n",(0,t.jsxs)(n.li,{children:["Two writes are still required when ",(0,t.jsx)(n.strong,{children:"version 2"})," is used."]}),"\n"]}),"\n",(0,t.jsxs)(n.p,{children:["The intent is for very small messages to be stored in the queue\nindex as an optimisation, and for all other messages to be\nwritten to the message store. This is controlled by the\nconfiguration item ",(0,t.jsx)("code",{children:"queue_index_embed_msgs_below"}),". By\ndefault, messages with a serialised size of less than 4096 bytes\n(including properties and headers) are stored in the queue\nindex."]}),"\n",(0,t.jsxs)(n.p,{children:["Each queue index needs to keep at least one segment file in\nmemory when reading messages from disk. The segment file\ncontains records for 16,384 messages. Therefore be cautious if\nincreasing ",(0,t.jsx)("code",{children:"queue_index_embed_msgs_below"}),"; a small\nincrease can lead to a large amount of memory used."]}),"\n",(0,t.jsx)(n.h2,{id:"limits",children:"OS and Runtime Limits Affecting"}),"\n",(0,t.jsx)(n.p,{children:"It is possible for persistence to underperform because the\npersister is limited in the number of file handles or async\nthreads it has to work with. In both cases this can happen when\nyou have a large number of queues which need to access the disk\nsimultaneously."}),"\n",(0,t.jsx)(n.h3,{id:"file-handles",children:"Too Few File Handles"}),"\n",(0,t.jsxs)(n.p,{children:["The RabbitMQ server is limited in the ",(0,t.jsx)(n.a,{href:"./networking#open-file-handle-limit",children:"number of file handles"})," it can open.\nEvery running network connection requires one file handle, and the rest are available\nfor queues to use. If there are more disk-accessing queues than\nfile handles after network connections have been taken into\naccount, then the disk-accessing queues will share the file\nhandles among themselves; each gets to use a file handle for a\nwhile before it is taken back and given to another queue."]}),"\n",(0,t.jsx)(n.p,{children:"This prevents the server from crashing due to there being too\nmany disk-accessing queues, but it can become expensive. The\nmanagement plugin can show I/O statistics for each node in the\ncluster; as well as showing rates of reads, writes, seeks and so\non it will also show a rate of file handle churn \u2014 the rate at\nwhich file handles are recycled in this way. A busy server with\ntoo few file handles might be doing hundreds of reopens per\nsecond - in which case its performance is likely to increase\nnotably if given more file handles."}),"\n",(0,t.jsx)(n.h2,{id:"msg-store-index-implementations",children:"Classic Queues v1: Alternate Message Store Index Implementations"}),"\n",(0,t.jsx)(n.p,{children:"As mentioned above, each message which is written to the message\nstore uses a small amount of memory for its index entry. The\nmessage store index used by classic queues v1 is pluggable in RabbitMQ, and other\nimplementations are available as plugins which can remove this\nlimitation."}),"\n",(0,t.jsx)(n.p,{children:"The reason they are not shipped with the RabbitMQ distribution is\nthat they all use native code. Note that such plugins typically\nmake the message store run more slowly."})]})}function u(e={}){let{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(h,{...e})}):h(e)}},28453(e,n,s){s.d(n,{R:()=>a,x:()=>o});var i=s(96540);let t={},r=i.createContext(t);function a(e){let n=i.useContext(r);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:a(e.components),i.createElement(r.Provider,{value:n},e.children)}}}]);