https://quickwit.io/assets/js/19ba6701.1831383a.js

1"use strict";(self.webpackChunkquickwit_io=self.webpackChunkquickwit_io||[]).push([[18453],{3905:(e,t,a)=>{a.d(t,{Zo:()=>u,kt:()=>g});var n=a(67294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?o(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):o(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function l(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},o=Object.keys(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),p=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},u=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},d="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},c=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=p(a),c=r,g=d["".concat(s,".").concat(c)]||d[c]||m[c]||o;return a?n.createElement(g,i(i({ref:t},u),{},{components:a})):n.createElement(g,i({ref:t},u))}));function g(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,i=new Array(o);i[0]=c;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[d]="string"==typeof e?e:r,i[1]=l;for(var p=2;p<o;p++)i[p]=a[p];return n.createElement.apply(null,i)}return n.createElement.apply(null,a)}c.displayName="MDXCreateElement"},90866:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>l,toc:()=>p});var n=a(87462),r=(a(67294),a(3905));const o={title:"Storage configuration",sidebar_position:2},i=void 0,l={unversionedId:"configuration/storage-config",id:"version-0.8.1/configuration/storage-config",title:"Storage configuration",description:"Supported Storage Providers",source:"@site/versioned_docs/version-0.8.1/configuration/storage-config.md",sourceDirName:"configuration",slug:"/configuration/storage-config",permalink:"/docs/0.8.1/configuration/storage-config",draft:!1,tags:[],version:"0.8.1",sidebarPosition:2,frontMatter:{title:"Storage configuration",sidebar_position:2},sidebar:"tutorialSidebar",previous:{title:"Node configuration",permalink:"/docs/0.8.1/configuration/node-config"},next:{title:"Index configuration",permalink:"/docs/0.8.1/configuration/index-config"}},s={},p=[{value:"Supported Storage Providers",id:"supported-storage-providers",level:2},{value:"Storage URIs",id:"storage-uris",level:2},{value:"Local file storage URIs",id:"local-file-storage-uris",level:3},{value:"Storage configuration",id:"storage-configuration",level:2},{value:"S3 storage configuration",id:"s3-storage-configuration",level:3},{value:"Environment variables",id:"environment-variables",level:4},{value:"Storage flavors",id:"storage-flavors",level:4},{value:"Azure storage configuration",id:"azure-storage-configuration",level:3},{value:"Environment variables",id:"environment-variables-1",level:4},{value:"Storage configuration examples for various object storage providers",id:"storage-configuration-examples-for-various-object-storage-providers",level:2},{value:"Garage",id:"garage",level:3},{value:"MinIO",id:"minio",level:3}],u={toc:p},d="wrapper";

1function m(e){let{components:t,...a}=e;return(0,r.kt)(d,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h2",{id:"supported-storage-providers"},"Supported Storage Providers"),(0,r.kt)("p",null,"Quickwit currently supports four types of storage providers:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Amazon S3 and S3-compatible (Garage, MinIO, ...)"),(0,r.kt)("li",{parentName:"ul"},"Azure Blob Storage"),(0,r.kt)("li",{parentName:"ul"},"Local file storage*"),(0,r.kt)("li",{parentName:"ul"},"Google Cloud Storage (native API)")),(0,r.kt)("h2",{id:"storage-uris"},"Storage URIs"),(0,r.kt)("p",null,'Storage URIs refer to different storage providers identified by a URI "protocol" or "scheme". Quickwit supports the following storage URI protocols:'),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"s3://")," for Amazon S3 and S3-compatible"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"azure://")," for Azure Blob Storage"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"file://")," for local file systems"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"gs://")," for Google Cloud Storage")),(0,r.kt)("p",null,"In general, you can use a storage URI or a file path anywhere you would intuitively expect a file path. For instance:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"when setting the ",(0,r.kt)("inlineCode",{parentName:"li"},"index_uri")," of an index to specify the storage provider and location;"),(0,r.kt)("li",{parentName:"ul"},"when setting the ",(0,r.kt)("inlineCode",{parentName:"li"},"metastore_uri")," in a node config to set up a file-backed metastore;"),(0,r.kt)("li",{parentName:"ul"},"when passing a file path as a command line argument.")),(0,r.kt)("h3",{id:"local-file-storage-uris"},"Local file storage URIs"),(0,r.kt)("p",null,"Quickwit interprets regular file paths as local file system URIs. Relative file paths are allowed and are resolved relatively to the current working directory (CWD). ",(0,r.kt)("inlineCode",{parentName:"p"},"~")," can be used as a shortcut to refer to the user\u2019s home directory. The following are valid local file system URIs:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-markdown"},"- /var/quickwit\n- file:///var/quickwit\n- /home/quickwit/data\n- ~/data\n- ./quickwit\n")),(0,r.kt)("admonition",{type:"caution"},(0,r.kt)("p",{parentName:"admonition"},"When using the ",(0,r.kt)("inlineCode",{parentName:"p"},"file://")," protocol, a third ",(0,r.kt)("inlineCode",{parentName:"p"},"/")," is necessary to express an absolute path. For instance, the following URI ",(0,r.kt)("inlineCode",{parentName:"p"},"file://home/quickwit/")," is interpreted as ",(0,r.kt)("inlineCode",{parentName:"p"},"./home/quickwit"))),(0,r.kt)("h2",{id:"storage-configuration"},"Storage configuration"),(0,r.kt)("p",null,"This section contains one configuration subsection per storage provider. If a storage configuration parameter is not explicitly set, Quickwit relies on the default values provided by the storage provider SDKs (",(0,r.kt)("a",{parentName:"p",href:"https://github.com/Azure/azure-sdk-for-rust"},"Azure SDK for Rust"),", ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/awslabs/aws-sdk-rust"},"AWS SDK for Rust"),")."),(0,r.kt)("h3",{id:"s3-storage-configuration"},"S3 storage configuration"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"flavor")),(0,r.kt)("td",{parentName:"tr",align:null},"The optional storage flavor to use. Available flavors are ",(0,r.kt)("inlineCode",{parentName:"td"},"digital_ocean"),", ",(0,r.kt)("inlineCode",{parentName:"td"},"garage"),", ",(0,r.kt)("inlineCode",{parentName:"td"},"gcs"),", and ",(0,r.kt)("inlineCode",{parentName:"td"},"minio"),"."),(0,r.kt)("td",{parentName:"tr",align:null})),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"access_key_id")),(0,r.kt)("td",{parentName:"tr",align:null},"The AWS access key ID."),(0,r.kt)("td",{parentName:"tr",align:null})),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"secret_access_key")),(0,r.kt)("td",{parentName:"tr",align:null},"The AWS secret access key."),(0,r.kt)("td",{parentName:"tr",align:null})),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"region")),(0,r.kt)("td",{parentName:"tr",align:null},"The AWS region to send requests to."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"us-east-1")," (SDK default)")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"endpoint")),(0,r.kt)("td",{parentName:"tr",align:null},"Custom endpoint for use with S3-compatible providers."),(0,r.kt)("td",{parentName:"tr",align:null},"SDK default")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"force_path_style_access")),(0,r.kt)("td",{parentName:"tr",align:null},"Disables ",(0,r.kt)("a",{parentName:"td",href:"https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html"},"virtual-hosted\u2013style")," requests. Required by some S3-compatible providers (Ceph, MinIO)."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"false"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"disable_multi_object_delete")),(0,r.kt)("td",{parentName:"tr",align:null},"Disables ",(0,r.kt)("a",{parentName:"td",href:"https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html"},"Multi-Object Delete")," requests. Required by some S3-compatible providers (GCS)."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"false"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"disable_multipart_upload")),(0,r.kt)("td",{parentName:"tr",align:null},"Disables ",(0,r.kt)("a",{parentName:"td",href:"https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html"},"multipart upload")," of objects. Required by some S3-compatible providers (GCS)."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"false"))))),(0,r.kt)("admonition",{type:"warning"},(0,r.kt)("p",{parentName:"admonition"},"Hardcoding credentials into configuration files is not secure and strongly discouraged. Prefer the alternative authentication methods that your storage backend may provide.")),(0,r.kt)("h4",{id:"environment-variables"},"Environment variables"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Env variable"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"QW_S3_ENDPOINT")),(0,r.kt)("td",{parentName:"tr",align:null},"Custom S3 endpoint.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"QW_S3_MAX_CONCURRENCY")),(0,r.kt)("td",{parentName:"tr",align:null},"Limit the number of concurrent requests to S3")))),(0,r.kt)("h4",{id:"storage-flavors"},"Storage flavors"),(0,r.kt)("p",null,"Storage flavors ensure that Quickwit works correctly with storage providers that deviate from the S3 API by automatically configuring the appropriate settings. The available flavors are:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"digital_ocean")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"garage")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"gcs")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"minio"))),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Digital Ocean")),(0,r.kt)("p",null,"The Digital Ocean flavor (",(0,r.kt)("inlineCode",{parentName:"p"},"digital_ocean"),") forces path-style access and turns off multi-object delete requests."),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Garage flavor")),(0,r.kt)("p",null,"The Garage flavor (",(0,r.kt)("inlineCode",{parentName:"p"},"garage"),") overrides the ",(0,r.kt)("inlineCode",{parentName:"p"},"region")," parameter to ",(0,r.kt)("inlineCode",{parentName:"p"},"garage")," and forces path-style access."),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Google Cloud Storage")),(0,r.kt)("p",null,"The Google Cloud Storage flavor (",(0,r.kt)("inlineCode",{parentName:"p"},"gcs"),") turns off multi-object delete requests and multipart uploads."),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"MinIO flavor")),(0,r.kt)("p",null,"The MinIO flavor (",(0,r.kt)("inlineCode",{parentName:"p"},"minio"),") forces path-style access."),(0,r.kt)("p",null,"Example of a storage configuration for Google Cloud Storage in YAML format:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"storage:\n  s3:\n    flavor: gcs\n    region: us-east1\n    endpoint: https://storage.googleapis.com\n")),(0,r.kt)("h3",{id:"azure-storage-configuration"},"Azure storage configuration"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"account")),(0,r.kt)("td",{parentName:"tr",align:null},"The Azure storage account name."),(0,r.kt)("td",{parentName:"tr",align:null})),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"access_key")),(0,r.kt)("td",{parentName:"tr",align:null},"The Azure storage account access key."),(0,r.kt)("td",{parentName:"tr",align:null})))),(0,r.kt)("h4",{id:"environment-variables-1"},"Environment variables"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Env variable"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"QW_AZURE_STORAGE_ACCOUNT")),(0,r.kt)("td",{parentName:"tr",align:null},"Azure Blob Storage account name.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"QW_AZURE_STORAGE_ACCESS_KEY")),(0,r.kt)("td",{parentName:"tr",align:null},"Azure Blob Storage account access key.")))),(0,r.kt)("p",null,"Example of a storage configuration for Azure in YAML format:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"storage:\n  azure:\n    account: your-azure-account-name\n    access_key: your-azure-access-key\n")),(0,r.kt)("h2",{id:"storage-configuration-examples-for-various-object-storage-providers"},"Storage configuration examples for various object storage providers"),(0,r.kt)("h3",{id:"garage"},"Garage"),(0,r.kt)("p",null,(0,r.kt)("a",{parentName:"p",href:"https://garagehq.deuxfleurs.fr/"},"Garage")," is an open-source distributed object storage service tailored for self-hosting."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"storage:\n  s3:\n    flavor: garage\n    endpoint: http://127.0.0.1:3900\n")),(0,r.kt)("h3",{id:"minio"},"MinIO"),(0,r.kt)("p",null,(0,r.kt)("a",{parentName:"p",href:"https://min.io/"},"MinIO")," is a high-performance object storage."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"storage:\n  s3:\n    flavor: minio\n    endpoint: http://127.0.0.1:9000\n")),(0,r.kt)("p",null,"Note: ",(0,r.kt)("inlineCode",{parentName:"p"}

1,"default_index_root_uri")," or index URIs do not include the endpoint, you should set it as a typical S3 path such as ",(0,r.kt)("inlineCode",{parentName:"p"},"s3://indexes"),"."))}m.isMDXComponent=!0}}]);