1"use strict";(self.webpackChunkdocusaurus=self.webpackChunkdocusaurus||[]).push([[1939],{5542:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>a,default:()=>h,frontMatter:()=>d,metadata:()=>s,toc:()=>c});var s=t(6879),i=t(4848),r=t(8453);const d={title:"Ceph - ext4 optimization for filestore",authors:"DCasier",description:"Filestore optimization",created:"2016/02/14 15:27:40",comment_status:"open",post_name:"ceph-ext4-optimisation-for-filestore",status:"publish",post_type:"post",image:"../img/post-images.jpg"},a=void 0,o={authorsImageUrls:[void 0]},c=[{value:"I/O optimization",id:"io-optimization",level:2},{value:"Ext4 : reducing IOPS",id:"ext4--reducing-iops",level:3},{value:"Sample :",id:"sample-",level:4},{value:"And ?",id:"and-",level:3},{value:"Howto",id:"howto",level:3},{value:"Step 1 : Determine Ext4 parameters and size of SSD partition",id:"step-1--determine-ext4-parameters-and-size-of-ssd-partition",level:4},{value:"Step 2 : Create SSD(10GiB)+HDD device and format",id:"step-2--create-ssd10gibhdd-device-and-format",level:4},{value:"That's it !",id:"thats-it-",level:5},{value:"Tests",id:"tests",level:3},{value:"Drawback",id:"drawback",level:3}];function l(e){const n={blockquote:"blockquote",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",h5:"h5",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Locale post-image",src:t(9878).A+"",width:"1024",height:"640"})}),"\n",(0,i.jsx)(n.h2,{id:"io-optimization",children:"I/O optimization"}),"\n",(0,i.jsx)(n.p,{children:"There is a way on ext4 to place a very high proportion of metadata on SSD and by doing that reducing IOPS on the drive. Let me introduce that."}),"\n",(0,i.jsx)(n.h3,{id:"ext4--reducing-iops",children:"Ext4 : reducing IOPS"}),"\n",(0,i.jsx)(n.p,{children:(0,i.jsx)(n.img,{alt:"Locale title",src:t(3955).A+"",width:"420",height:"374"})}),"\n",(0,i.jsx)(n.p,{children:"In i_block, Access block can be direct or indirect. If less than 3 extents (3*15 bytes) are needed for directory content, everything is placed in the inode (and direct adressing)."}),"\n",(0,i.jsx)(n.h4,{id:"sample-",children:"Sample :"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-bash",children:"> [root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R 'htree current/'\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nRoot node dump:\nReserved zero: 0\nHash Version: 1\nInfo length: 8\nIndirect levels: 0\nFlags: 0\nNumber of entries (count): 16\nNumber of entries (limit): 508\nEntry #0: Hash 0x00000000, block 1\nEntry #1: Hash 0x11371de6, block 9\nEntry #2: Hash 0x25acde12, block 5\nEntry #3: Hash 0x3630f220, block 12\nEntry #4: Hash 0x463ad436, block 3\nEntry #5: Hash 0x54285f38, block 11\nEntry #6: Hash 0x62cd9430, block 6\nEntry #7: Hash 0x74515ea8, block 10\nEntry #8: Hash 0x8440daf8, block 2\nEntry #9: Hash 0x96cd838c, block 13\nEntry #10: Hash 0xa781eb78, block 7\nEntry #11: Hash 0xb69832e6, block 14\nEntry #12: Hash 0xc61cf90e, block 4\nEntry #13: Hash 0xd3c3a084, block 16\nEntry #14: Hash 0xe155199e, block 8\nEntry #15: Hash 0xf01efb38, block 15\nEntry #0: Hash 0x00000000, block 1\nReading directory block 1, phys 1605606\n13975631 0x087153ae-82a1bcf7 (20) 11.1717_head\n232 0x08a2fe5c-c61778e0 (20) 12.19a_head\n11550030 0x00dff92e-878fc555 (20) 11.1eb4_head\n(\u2026)\n"})}),"\n",(0,i.jsx)(n.p,{children:'Access to "current/" is not direct because there are too many directories / files in "current/" and need htree.'}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-bash",children:"> [root@XXXXX ceph]# debugfs /dev/mapper/dm-sde6-sda -R 'htree current/11.beb_head/DIR_B/DIR_E/DIR_B/DIR_E/DIR_F'\n[root@XXXXX ceph]# debugfs /dev/mapper/dm-sde6-sda -R \u2018htree current/11.beb_head/DIR_B/DIR_E/DIR_B/DIR_E/DIR_F\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nRoot node dump:\nReserved zero: 0\nHash Version: 1\nInfo length: 8\nIndirect levels: 0\nFlags: 0\nNumber of entries (count): 2\nNumber of entries (limit): 508\nEntry #0: Hash 0x00000000, block 1\nEntry #1: Hash 0x8eb23f6c, block 2\nEntry #0: Hash 0x00000000, block 1\nReading directory block 1, phys 28332092\n0 0x0c1e3712-ceb03663 (2644) default.12269.111\\u6d46ea490b9e361ab1a0679df41fae0eac56cabd.swf__head_68BFEBEB__b\n13897999 0x8b6710dc-5a024832 (16) DIR_0\n13898001 0x0380573a-2be3ad2e (16) DIR_3\n13898004 0x5ccf4a4e-fd506286 (16) DIR_6\n13898005 0x25a6bbb8-ccc71946 (16) DIR_7\n13898006 0x76c3db98-6925c6a4 (16) DIR_8\n13898007 0x6f4e207e-12569d4
10 (16) DIR_9\n13898009 0x7deaea4e-dfd500ce (1356) DIR_C\nEntry #1: Hash 0x8eb23f6c, block 2\nReading directory block 2, phys 28332093\n0 0x8eb23f6c-20e729d0 (2644) default.12269.245\\uf3caec303396f714cd386516079292e0dd7d5936.swf__head_96DFEBEB__b\n13898000 0x9772ae86-7a15d85e (16) DIR_1\n"})}),"\n",(0,i.jsx)(n.p,{children:'You see here that access to "current/11.beb_head/DIR_B/DIR_E/DIR_B/DIR_E/DIR_F" is not direct from "current/11.beb_head/DIR_B/DIR_E/DIR_B/DIR_E".'}),"\n",(0,i.jsx)(n.p,{children:"What does that mean ?"}),"\n",(0,i.jsxs)(n.p,{children:["The virtual view is (strace on ceph-osd process) :\n",(0,i.jsx)(n.img,{alt:"Locale title",src:t(8505).A+"",width:"1198",height:"143"})]}),"\n",(0,i.jsxs)(n.p,{children:["But I/O profil :\n",(0,i.jsx)(n.img,{alt:"Locale title",src:t(9786).A+"",width:"1271",height:"143"}),"\nWith this parameter, there are more directories but less IOPS on drive :"]}),"\n",(0,i.jsxs)(n.blockquote,{children:["\n",(0,i.jsx)(n.p,{children:"filestore_merge_threshold = 2 filestore_split_multiple = 1"}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"Max 32 objects in a directory and all is contained in 3 extents in the inode. Demo :"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-bash",children:"> [root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R 'htree current/11.1717_head'\n[root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n[root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head/DIR_7\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n[root@ao24341 ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head/DIR_7/DIR_1\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n[root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head/DIR_7/DIR_1/DIR_7\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n[root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head/DIR_7/DIR_1/DIR_7/DIR_1\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n[root@XXXXX ~]# debugfs /dev/mapper/dm-sdi5-sda -R \u2018htree current/11.1717_head/DIR_7/DIR_1/DIR_7/DIR_1/DIR_1\u2019\ndebugfs 1.42.13.wc4 (28-Nov-2015)\nhtree: Not a hash-indexed directory\n"})}),"\n",(0,i.jsx)(n.h3,{id:"and-",children:"And ?"}),"\n",(0,i.jsxs)(n.p,{children:["With Ext4, you can place the allocation bitmaps and the inode table at the beginning of the disk (",(0,i.jsx)(n.strong,{children:"packed_meta_blocks"})," option)."]}),"\n",(0,i.jsxs)(n.p,{children:["If you create a SSH+HDD concatenation, you can store all metadata (inode and xattr) on the SSD :\n",(0,i.jsx)(n.img,{alt:"Locale title",src:t(2824).A+"",width:"284",height:"673"})]}),"\n",(0,i.jsx)(n.h3,{id:"howto",children:"Howto"}),"\n",(0,i.jsx)(n.h4,{id:"step-1--determine-ext4-parameters-and-size-of-ssd-partition",children:"Step 1 : Determine Ext4 parameters and size of SSD partition"}),"\n",(0,i.jsx)(n.p,{children:"Informations :"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:"Inode size : 256 bytes"}),"\n",(0,i.jsx)(n.li,{children:"HDD size : 4 TB"}),"\n",(0,i.jsx)(n.li,{children:"Average file size on production > 128 KiB"}),"\n",(0,i.jsx)(n.li,{children:"128 MiB per Block Group\nParameters :"}),"\n",(0,i.jsx)(n.li,{children:"Bytes-per-inodes (-i) : 131072 (128KiB) [Default : 4 KiB]"}),"\n",(0,i.jsxs)(n.li,{children:["number-of-groups (-G) : 32768 (max adressing 4,4 TB [128",(0,i.jsx)(n.em,{children:"1024"}),"1024*32768])\nSize of SSD partition :"]}),"\n",(0,i.jsx)(n.li,{children:"Max of inodes : 30517578 (4000000000000/131072) [HDD size / Bytes-per-inodes]"}),"\n",(0,i.jsx)(n.li,{children:"Size of inode table : < 7,5 GiB (30517578*256/1024/1024/1024) [Max inodes * Inode size]\nSize of SSD partition with default bytes-per-inodes :"}),"\n",(0,i.jsx)(n.li,{children:"Inodes : 976562500 (4000000000000/4096)"}),"\n",(0,i.jsx)(n.li,{children:"Size of inode table : ~ 234 GiB (976562500*256/1024/1024/1024)"}),"\n"]}),"\n",(0,i.jsx)(n.h4,{id:"step-2--create-ssd10gibhdd-device-and-format",children:"Step 2 : Create SSD(10GiB)+HDD device and format"}),"\n",(0,i.jsx)(n.p,{children:"With 2 disks, LVM strips data. We need to create it manually with dmcreate"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-bash",children:" size_of() {\n blockdev \u2013getsize $1\n }
1\n\n mkdmsetup() {\n _ssd=/dev/$1\n _hdd=/dev/$2\n _size_of_ssd=$(size_of $_ssd)\n echo \u201c\u201d\u201d0 $_size_of_ssd linear $_ssd 0\n $_size_of_ssd $(size_of $_hdd) linear $_hdd 0\u201d | dmsetup create dm-$(echo ${1}|awk -F\u2019/\u2019 \u2018{print $NF}\u2019)-$(echo ${2}|awk -F\u2019/\u2019 \u2018{print $NF}\u2019)\n }\n\n mkdmsetup sde1 sdb\n"})}),"\n",(0,i.jsx)(n.p,{children:"Format"}),"\n",(0,i.jsx)(n.pre,{children:(0,i.jsx)(n.code,{className:"language-bash",children:"> mkfs.ext4 -F -O ^has_journal,^uninit_bg -E packed_meta_blocks=1,nodiscard,lazy_itable_init=0 -G 32768 -i 131072 /dev/mapper/dm-sde1-sdb\n"})}),"\n",(0,i.jsx)(n.h5,{id:"thats-it-",children:"That's it !"}),"\n",(0,i.jsx)(n.h3,{id:"tests",children:"Tests"}),"\n",(0,i.jsx)(n.p,{children:"1 PG with 9500 objects"}),"\n",(0,i.jsxs)(n.blockquote,{children:["\n",(0,i.jsx)(n.p,{children:"[Demo]#find current/11.1af4_head |wc -l 11181 [Demo]#find current/11.1af4_head -type f |wc -l 9507"}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"First, iostat"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:"SSD : 2240 IOPS"}),"\n",(0,i.jsx)(n.li,{children:"HDD : 400 IOPS"}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"Find + getfattr in 30 sec"}),"\n",(0,i.jsxs)(n.blockquote,{children:["\n",(0,i.jsxs)(n.p,{children:["[Demo]# time find current/11.1af4_head -exec getfattr '","' ; >/dev/null real 0m29.601s user\n0m1.133s sys 0m6.959s"]}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"And 3/4 of the I/O in SSD SSD"}),"\n",(0,i.jsxs)(n.blockquote,{children:["\n",(0,i.jsx)(n.p,{children:"[Demo]# blktrace -d /dev/sdf -o - |blkparse -i - |grep ' Q ' > /tmp/blkparse.sdf [Demo]# wc -l /tmp/blkparse.sdf 31511 /tmp/blkparse.sdf"}),"\n"]}),"\n",(0,i.jsx)(n.p,{children:"HDD"}),"\n",(0,i.jsxs)(n.blockquote,{children:["\n",(0,i.jsx)(n.p,{children:"[Demo]# blktrace -d /dev/sdg -o - |blkparse -i - |grep ' Q ' > /tmp/blkparse.sdg [Demo]# wc -l /tmp/blkparse.sdg 11050 /tmp/blkparse.sdg"}),"\n"]}),"\n",(0,i.jsx)(n.h3,{id:"drawback",children:"Drawback"}),"\n",(0,i.jsx)(n.p,{children:"With this model, filestore rearrange the tree very frequently : + 40 I/O every 32 objects link/unlink"})]})}function h(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,i.jsx)(n,{...e,children:(0,i.jsx)(l,{...e})}):l(e)}},9786:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/Real-access-767c3b773a247916c4464387588d63fd.png"},2824:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/SSD_HDD-1-f917e8320cefd9bdbbc25679cfc10148.png"},8505:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/Virtual-access-20ebf08ddaf7375e3f3a10711c341187.png"},3955:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/extent_tree_layout-large-be392df73bf2f0936525f20ac9a813f0.jpg"},9878:(e,n,t)=>{t.d(n,{A:()=>s});const s=t.p+"assets/images/post-images-2c61e6058b7a1d04c516bbead7de6f79.jpg"},8453:(e,n,t)=>{t.d(n,{R:()=>d,x:()=>a});var s=t(6540);const i={},r=s.createContext(i);function d(e){const n=s.useContext(r);return s.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:d(e.components),s.createElement(r.Provider,{value:n},e.children)}},6879:e=>{e.exports=JSON.parse('{"permalink":"/blog/2016/02/14/ceph-ext4-optimisation-for-filestore","editUrl":"https://git.aevoo.com/aevoo-doc/aevoo-blog/src/branch/main/blog/2016/02-14-ceph-ext4-optimisation-for-filestore.md","source":"@site/blog/2016/02-14-ceph-ext4-optimisation-for-filestore.md","title":"Ceph - ext4 optimization for filestore","description":"Filestore optimization","date":"2016-02-14T00:00:00.000Z","tags":[],"hasTruncateMarker":true,"authors":[{"name":"David Casier","key":"DCasier","page":null}],"frontMatter":{"title":"Ceph - ext4 optimization for filestore","authors":"DCasier","description":"Filestore optimization","created":"2016/02/14 15:27:40","comment_status":"open","post_name":"ceph-ext4-optimisation-for-filestore","status":"publish","post_type":"post","image":"../img/post-images.jpg"},"unlisted":false,"prevItem":{"title":"Ceph - Firewalld zones and services","permalink":"/blog/2016/02/25/ceph-firewalld-zones-and-services"}}')}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.