1"use strict";(globalThis.webpackChunkcachelib=globalThis.webpackChunkcachelib||[]).push([[4869],{15680(e,r,t){t.r(r),t.d(r,{MDXContext:()=>c,MDXProvider:()=>p,mdx:()=>h,useMDXComponents:()=>m,withMDXComponents:()=>d});var n=t(96540);function a(e,r,t){return r in e?Object.defineProperty(e,r,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[r]=t,e}function o(){return o=Object.assign||function(e){for(var r=1;r<arguments.length;r++){var t=arguments[r];for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])}return e},o.apply(this,arguments)}function i(e,r){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);r&&(n=n.filter(function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable})),t.push.apply(t,n)}return t}function s(e){for(var r=1;r<arguments.length;r++){var t=null!=arguments[r]?arguments[r]:{};r%2?i(Object(t),!0).forEach(function(r){a(e,r,t[r])}):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach(function(r){Object.defineProperty(e,r,Object.getOwnPropertyDescriptor(t,r))})}return e}function l(e,r){if(null==e)return{};var t,n,a=function(e,r){if(null==e)return{};var t,n,a={},o=Object.keys(e);for(n=0;n<o.length;n++)t=o[n],r.indexOf(t)>=0||(a[t]=e[t]);return a}(e,r);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)t=o[n],r.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var c=n.createContext({}),d=function(e){return function(r){var t=m(r.components);return n.createElement(e,o({},r,{components:t}))}},m=function(e){var r=n.useContext(c),t=r;return e&&(t="function"==typeof e?e(r):s(s({},r),e)),t},p=function(e){var r=m(e.components);return n.createElement(c.Provider,{value:r},e.children)},f={inlineCode:"code",wrapper:function(e){var r=e.children;return n.createElement(n.Fragment,{},r)}},u=n.forwardRef(function(e,r){var t=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=m(t),p=a,u=d["".concat(i,".").concat(p)]||d[p]||f[p]||o;return t?n.createElement(u,s(s({ref:r},c),{},{components:t})):n.createElement(u,s({ref:r},c))});function h(e,r){var t=arguments,a=r&&r.mdxType;if("string"==typeof e||a){var o=t.length,i=new Array(o);i[0]=u;var s={};for(var l in r)hasOwnProperty.call(r,l)&&(s[l]=r[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var c=2;c<o;c++)i[c]=t[c];return n.createElement.apply(null,i)}return n.createElement.apply(null,t)}u.displayName="MDXCreateElement"},99975(e,r,t){t.r(r),t.d(r,{assets:()=>l,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>s,toc:()=>c});var n=t(58168),a=(t(96540),t(15680));const o={id:"Cross_Host_Persistence_APIs_Internal",title:"Cross Host Persistence APIs (internal)"},i=void 0,s={unversionedId:"facebook/Cache_Persistence/Cross_Host_Persistence_APIs_Internal",id:"facebook/Cache_Persistence/Cross_Host_Persistence_APIs_Internal",title:"Cross Host Persistence APIs (internal)",description:"Introduction",source:"@site/docs/facebook/Cache_Persistence/cross_host_persistence_APIs_internal.md",sourceDirName:"facebook/Cache_Persistence",slug:"/facebook/Cache_Persistence/Cross_Host_Persistence_APIs_Internal",permalink:"/docs/facebook/Cache_Persistence/Cross_Host_Persistence_APIs_Internal",draft:!1,editUrl:"https://github.com/facebook/CacheLib/edit/main/website/docs/facebook/Cache_Persistence/cross_host_persistence_APIs_internal.md",tags:[],version:"current",frontMatter:{id:"Cross_Host_Persistence_APIs_Internal",title:"Cross Host Persistence APIs (internal)"}},l={},c=[{value:"Introduction",id:"introduction",level:2},{value:"How To Use Manifold Reader And Writer",id:"how-to-use-manifold-reader-and-writer",level:2},{value:"Ture The Performance",id:"ture-the-performance",level:2}],d={toc:c};function m({components:e,...r}){return(0,a.mdx)("wrapper",(0,n.A)({},d,r,{components:e,mdxType:"MDXLayout"}),(0,a.mdx)("h2",{id:"introduction"},"Introduction"),(0,a.mdx)("p",null,"Cachelib cross host persistence feature requires user to provide customized stream reader and writer to communicate with backend remote storage. ",(0,a.mdx)("a",{parentName:"p",href:"https://www.internalfb.com/intern/wiki/Manifold/Manifold_Overview/"},"Manifold")," is widely used inside FB as remote storage, and it is most likely to be the first choice of FB users, so Cachelib has provided built-in Manifold reader and writer."),(0,a.mdx)("h2",{id:"how-to-use-manifold-reader-and-writer"},"How To Use Manifold Reader And Writer"),(0,a.mdx)("p",null,(0,a.mdx)("a",{parentName:"p",href:"https://www.internalfb.com/diff/D29218346"},"Reader"),":"),(0,a.mdx)("pre",null,(0,a.mdx)("code",{parentName:"pre",className:"language-cpp"},'PersistenceManager manager(cacheConfig);\nManifoldStreamReader reader(\n "bucket name",\n "object path",\n std::chrono::seconds{5000}, // timeout\n std::thread::hardware_concurrency() - 1, //parallelDownloads\n "", //apiKey\n "", //localPath\n 500 * 1024 * 1024 //bufferSize 500MB\n);\nmanager.restoreCache(reader);\n')),(0,a.mdx)("p",null,(0,a.mdx)("a",{parentName:"p",href:"https://www.internalfb.com/diff/D29218345"},"Writer"),":"),(0,a.mdx)("pre",null,(0,a.mdx)("code",{parentName:"pre",className:"language-cpp"},'PersistenceManager manager(cacheConfig);\nManifoldStreamWriter writer(\n "bucket name",\n "directory path",\n "filename",\n true, //userData\n std::chrono::seconds{5000}, //timeout\n std::chrono::seconds{50000}, //ttl\n std::thread::hardware_concurrency() - 1, //parallelUploads\n "", //apiKey\n "", //localPath\n 500 * 1024 * 1024, //bufferSize 500MB\n );\nmanager.saveCache(writer);\nwriter.close();\n')),(0,a.mdx)("p",null,(0,a.mdx)("a",{parentName:"p",href:"https://www.internalfb.com/diff/D28423748"},"Example"),"."),(0,a.mdx)("h2",{id:"ture-the-performance"},"Ture The Performance"),(0,a.mdx)("p",null,(0,a.mdx)("strong",{parentName:"p"},"Buffer Mode")," in reader downloads part of data to memory from manifold for each time."),(0,a.mdx)("p",null,(0,a.mdx)("strong",{parentName:"p"},"Buffer mode")," in writer uploads data to a separate temporary object in manifold, and concat all temp objects to the target at the end. The temporary objects have fixed 1 day expiry time, so the persistence must finish within 1 day."),(0,a.mdx)("p",null,"This is the default by giving empty localPath in reader/writer constructor, tune ",(0,a.mdx)("strong",{parentName:"p"},"bufferSize")," to get better performance."),(0,a.mdx)("p",null,(0,a.mdx)("strong",{parentName:"p"},"File mode")," in reader downloads the whole file to a local file from manifold at the beginning, and reads a part from the file into memory each time."),(0,a.mdx)("p",null,(0,a.mdx)("strong",{parentName:"p"},"File mode")," in writer persists data to a local file, and upload the file to manifold at the end."),(0,a.mdx)("p",null,"Provide a ",(0,a.mdx)("strong",{parentName:"p"},"localPath")," to enable this mode. The bufferSize doesn\u2019t play a big role in performance but helps in memory usage. User need to ensure no other thread/process is mutating the given file."),(0,a.mdx)("p",null,"To get better performance, user can compare File mode and Buffer mode with various buffer size. Based on previous testing, Buffer mode with large buffer size gives the best performance, while File mode perform better than a small buffer in Buffer mode."),(0,a.mdx)("p",null,(0,a.mdx)("inlineCode",{parentName:"p"},"parallelDownloads")," / ",(0,a.mdx)("inlineCode",{parentName:"p"},"parallelUploads")," are the number of thread Manifold client will use for uploading/downloading objects, more threads gives better performance. Reader and writer will download or upload objects asynchronously in buffer mode, so reader will prepare next data block while the current one is still using by PersistenceManager; writer will return immediately (except flushing) to PersistenceManager and uploads in background."))}m.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.