1"use strict";(self.webpackChunkquickwit_io=self.webpackChunkquickwit_io||[]).push([[27949],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>k});var a=n(67294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?i(Object(n),!0).forEach((function(t){r(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):i(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,a,r=function(e,t){if(null==e)return{};var n,a,r={},i=Object.keys(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a<i.length;a++)n=i[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),m=p(n),c=r,k=m["".concat(s,".").concat(c)]||m[c]||d[c]||i;return n?a.createElement(k,o(o({ref:t},u),{},{components:n})):a.createElement(k,o({ref:t},u))}));function k(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=c;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[m]="string"==typeof e?e:r,o[1]=l;for(var p=2;p<i;p++)o[p]=n[p];return a.createElement.apply(null,o)}return a.createElement.apply(null,n)}c.displayName="MDXCreateElement"},48652:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var a=n(87462),r=(n(67294),n(3905));const i={title:"Source configuration",sidebar_position:5},o=void 0,l={unversionedId:"configuration/source-config",id:"version-0.7.1/configuration/source-config",title:"Source configuration",description:"Quickwit can insert data into an index from one or multiple sources.",source:"@site/versioned_docs/version-0.7.1/configuration/source-config.md",sourceDirName:"configuration",slug:"/configuration/source-config",permalink:"/docs/0.7.1/configuration/source-config",draft:!1,tags:[],version:"0.7.1",sidebarPosition:5,frontMatter:{title:"Source configuration",sidebar_position:5},sidebar:"tutorialSidebar",previous:{title:"Metastore configuration",permalink:"/docs/0.7.1/configuration/metastore-config"},next:{title:"Ports configuration",permalink:"/docs/0.7.1/configuration/ports-config"}},s={},p=[{value:"Source ID",id:"source-id",level:2},{value:"Source type",id:"source-type",level:2},{value:"Source parameters",id:"source-parameters",level:2},{value:"File source (CLI only)",id:"file-source-cli-only",level:3},{value:"Ingest API source",id:"ingest-api-source",level:3},{value:"Kafka source",id:"kafka-source",level:3},{value:"Kafka source parameters",id:"kafka-source-parameters",level:4},{value:"Kinesis source",id:"kinesis-source",level:3},{value:"Pulsar source",id:"pulsar-source",level:3},{value:"Maximum number of pipelines per indexer",id:"maximum-number-of-pipelines-per-indexer",level:2},{value:"Desired number of pipelines",id:"desired-number-of-pipelines",level:2},{value:"Transform parameters",id:"transform-parameters",level:2},{value:"Input format",id:"input-format",level:2},{value:"Enabling/Disabling a source from an index",id:"enablingdisabling-a-source-from-an-index",level:2},{value:"Deleting a source from an index",id:"deleting-a-source-from-an-index",level:2}],u={toc:p},m="wrapper";
1function d(e){let{components:t,...n}=e;return(0,r.kt)(m,(0,a.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("p",null,"Quickwit can insert data into an index from one or multiple sources.\nA source can be added after index creation using the ",(0,r.kt)("a",{parentName:"p",href:"/docs/0.7.1/reference/cli#source"},"CLI command")," ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit source create"),".\nIt can also be enabled or disabled with the ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit source enable/disable")," subcommands."),(0,r.kt)("p",null,"A source is declared using an object called source config, which defines the source's settings. It consists of multiple parameters:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"source ID"),(0,r.kt)("li",{parentName:"ul"},"source type"),(0,r.kt)("li",{parentName:"ul"},"source parameters"),(0,r.kt)("li",{parentName:"ul"},"input_format"),(0,r.kt)("li",{parentName:"ul"},"maximum number of pipelines per indexer (optional)"),(0,r.kt)("li",{parentName:"ul"},"desired number of pipelines (optional)"),(0,r.kt)("li",{parentName:"ul"},"transform parameters (optional)")),(0,r.kt)("h2",{id:"source-id"},"Source ID"),(0,r.kt)("p",null,"The source ID is a string that uniquely identifies the source within an index. It may only contain uppercase or lowercase ASCII letters, digits, hyphens (",(0,r.kt)("inlineCode",{parentName:"p"},"-"),"), and underscores (",(0,r.kt)("inlineCode",{parentName:"p"},"_"),"). Finally, it must start with a letter and contain at least 3 characters but no more than 255."),(0,r.kt)("h2",{id:"source-type"},"Source type"),(0,r.kt)("p",null,"The source type designates the kind of source being configured. As of version 0.5, available source types are ",(0,r.kt)("inlineCode",{parentName:"p"},"ingest-api"),", ",(0,r.kt)("inlineCode",{parentName:"p"},"kafka"),", ",(0,r.kt)("inlineCode",{parentName:"p"},"kinesis"),", and ",(0,r.kt)("inlineCode",{parentName:"p"},"pulsar"),". The ",(0,r.kt)("inlineCode",{parentName:"p"},"file")," type is also supported but only for local ingestion from ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/reference/cli#tool-local-ingest"},"the CLI"),"."),(0,r.kt)("h2",{id:"source-parameters"},"Source parameters"),(0,r.kt)("p",null,"The source parameters indicate how to connect to a data store and are specific to the source type."),(0,r.kt)("h3",{id:"file-source-cli-only"},"File source (CLI only)"),(0,r.kt)("p",null,"A file source reads data from a local file. The file must consist of JSON objects separated by a newline (NDJSON).\nAs of version 0.5, a file source can only be ingested with the ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/reference/cli#tool-local-ingest"},"CLI command"),". Compressed files (bz2, gzip, ...) and remote files (Amazon S3, HTTP, ...) are not supported."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"./quickwit tool local-ingest --input-path <INPUT_PATH>\n")),(0,r.kt)("h3",{id:"ingest-api-source"},"Ingest API source"),(0,r.kt)("p",null,"An ingest API source reads data from the ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/reference/rest-api#ingest-data-into-an-index"},"Ingest API"),". This source is automatically created at the index creation and cannot be deleted nor disabled."),(0,r.kt)("h3",{id:"kafka-source"},"Kafka source"),(0,r.kt)("p",null,"A Kafka source reads data from a Kafka stream. Each message in the stream must hold a JSON object."),(0,r.kt)("p",null,"A tutorial is available ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/ingest-data/kafka"},"here"),"."),(0,r.kt)("h4",{id:"kafka-source-parameters"},"Kafka source parameters"),(0,r.kt)("p",null,"The Kafka source consumes a ",(0,r.kt)("inlineCode",{parentName:"p"},"topic")," using the client library ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/edenhill/librdkafka"},"librdkafka")," and forwards the key-value pairs carried by the parameter ",(0,r.kt)("inlineCode",{parentName:"p"},"client_params")," to the underlying librdkafka consumer. Common ",(0,r.kt)("inlineCode",{parentName:"p"},"client_params")," options are bootstrap servers (",(0,r.kt)("inlineCode",{parentName:"p"},"bootstrap.servers"),"), or security protocol (",(0,r.kt)("inlineCode",{parentName:"p"},"security.protocol"),"). Please, refer to ",(0,r.kt)("a",{parentName:"p",href:"https://kafka.apache.org/documentation/#consumerconfigs"},"Kafka")," and ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"},"librdkafka")," documentation pages for more advanced options."),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"topic")),(0,r.kt)("td",{parentName:"tr",align:null},"Name of the topic to consume."),(0,r.kt)("td",{parentName:"tr",align:null},"required")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"client_log_level")),(0,r.kt)("td",{parentName:"tr",align:null},"librdkafka client log level. Possible values are: debug, info, warn, error."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"info"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"client_params")),(0,r.kt)("td",{parentName:"tr",align:null},"librdkafka client configuration parameters."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"{}"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"enable_backfill_mode")),(0,r.kt)("td",{parentName:"tr",align:null},"Backfill mode stops the source after reaching the end of the topic."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"false"))))),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Kafka client parameters")),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"bootstrap.servers"),"\nComma-separated list of host and port pairs that are the addresses of a subset of the Kafka brokers in the Kafka cluster.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"auto.offset.reset"),"\nDefines the behavior of the source when consuming a partition for which there is no initial offset saved in the checkpoint. ",(0,r.kt)("inlineCode",{parentName:"p"},"earliest")," consumes from the beginning of the partition, whereas ",(0,r.kt)("inlineCode",{parentName:"p"},"latest")," (default) consumes from the end.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"enable.auto.commit"),"\nThe Kafka source manages commit offsets manually using the ",(0,r.kt)("a",{parentName:"p",href:"/docs/0.7.1/overview/concepts/indexing#checkpoint"},"checkpoint API")," and disables auto-commit.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"group.id"),"\nKafka-based distributed indexing relies on consumer groups. Unless overridden in the client parameters, the default group ID assigned to each consumer managed by the source is ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit-{index_uid}-{source_id}"),".")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"max.poll.interval.ms"),"\nShort max poll interval durations may cause a source to crash when back pressure from the indexer occurs. Therefore, Quickwit recommends using the default value of ",(0,r.kt)("inlineCode",{parentName:"p"},"300000")," (5 minutes)."))),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Adding a Kafka source to an index with the ",(0,r.kt)("a",{parentName:"em",href:"/docs/0.7.1/reference/cli#source"},"CLI"))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"cat << EOF > source-config.yaml\nversion: 0.7\nsource_id: my-kafka-source\nsource_type: kafka\nmax_num_pipelines_per_indexer: 1\ndesired_num_pipelines: 2\nparams:\n topic: my-topic\n client_params:\n bootstrap.servers: localhost:9092\n security.protocol: SSL\nEOF\n./quickwit source create --index my-index --source-config source-config.yaml\n")),(0,r.kt)("h3",{id:"kinesis-source"},"Kinesis source"),(0,r.kt)("p",null,"A Kinesis source reads data from an ",(0,r.kt)("a",{parentName:"p",href:"https://aws.amazon.com/kinesis/"},"Amazon Kinesis")," stream. Each message in the stream must hold a JSON object."),(0,r.kt)("p",null,"A tutorial is available ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/ingest-data/kinesis"},"here"),"."),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Kinesis source parameters")),(0,r.kt)("p",null,"The Kinesis source consumes a stream identified by a ",(0,r.kt)("inlineCode",{parentName:"p"},"stream_name")," and a ",(0,r.kt)("inlineCode",{parentName:"p"},"region"),"."),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"stream_name")),(0,r.kt)("td",{parentName:"tr",align:null},"Name of the stream to consume."),(0,r.kt)("td",{parentName:"tr",align:null},"required")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"region")),(0,r.kt)("td",{parentName:"tr",align:null},"The AWS region of the stream. Mutually exclusive with ",(0,r.kt)("inlineCode",{parentName:"td"},"endpoint"),"."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"us-east-1"))),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"endpoint")),(0,r.kt)("td",{parentName:"tr",align:null},"Custom endpoint for use with AWS-compatible Kinesis service. Mutually exclusive with ",(0,r.kt)("inlineCode",{parentName:"td"},"region"),"."),(0,r.kt)("td",{parentName:"tr",align:null},"optional")))),(0,r.kt)("p",null,"If no region is specified, Quickwit will attempt to find one in multiple other locations and with the following order of precedence:"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Environment variables (",(0,r.kt)("inlineCode",{parentName:"p"},"AWS_REGION")," then ",(0,r.kt)("inlineCode",{parentName:"p"},"AWS_DEFAULT_REGION"),")")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Config file, typically located at ",(0,r.kt)("inlineCode",{parentName:"p"},"~/.aws/config")," or otherwise specified by the ",(0,r.kt)("inlineCode",{parentName:"p"},"AWS_CONFIG_FILE")," environment variable if set and not empty.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Amazon EC2 instance metadata service determining the region of the currently running Amazon EC2 instance.")),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Default value: ",(0,r.kt)("inlineCode",{parentName:"p"},"us-east-1")))),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Adding a Kinesis source to an index with the ",(0,r.kt)("a",{parentName:"em",href:"/docs/0.7.1/reference/cli#source"},"CLI"))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"cat << EOF > source-config.yaml\nversion: 0.7\nsource_id: my-kinesis-source\nsource_type: kinesis\nparams:\n stream_name: my-stream\nEOF\nquickwit source create --index my-index --source-config source-config.yaml\n")),(0,r.kt)("h3",{id:"pulsar-source"},"Pulsar source"),(0,r.kt)("p",null,"A Puslar source reads data from one or several Pulsar topics. Each message in topic(s) must hold a JSON object."),(0,r.kt)("p",null,"A tutorial is available ",(0,r.kt)("a",{parentName:"p",href:"/docs/main-branch/ingest-data/pulsar"},"here"),"."),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Pulsar source parameters")),(0,r.kt)("p",null,"The Pulsar source consumes ",(0,r.kt)("inlineCode",{parentName:"p"},"topics")," using the client library ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/streamnative/pulsar-rs"},"pulsar-rs"),"."),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"topics")),(0,r.kt)("td",{parentName:"tr",align:null},"List of topics to consume."),(0,r.kt)("td",{parentName:"tr",align:null},"required")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"address")),(0,r.kt)("td",{parentName:"tr",align:null},"Pulsar URL (pulsar:// and pulsar+ssl://)."),(0,r.kt)("td",{parentName:"tr",align:null},"required")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"consumer_name")),(0,r.kt)("td",{parentName:"tr",align:null},"The consumer name to register with the pulsar source."),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"quickwit"))))),(0,r.kt)("p",null,(0,r.kt)("em",{parentName:"p"},"Adding a Pulsar source to an index with the ",(0,r.kt)("a",{parentName:"em",href:"/docs/0.7.1/reference/cli#source"},"CLI"))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"cat << EOF > source-config.yaml\nversion: 0.7\nsource_id: my-pulsar-source\nsource_type: pulsar\nparams:\n topics:\n - my-topic\n address: pulsar://localhost:6650\nEOF\n./quickwit source create --index my-index --source-config source-config.yaml\n")),(0,r.kt)("h2",{id:"maximum-number-of-pipelines-per-indexer"},"Maximum number of pipelines per indexer"),(0,r.kt)("p",null,"The ",(0,r.kt)("inlineCode",{parentName:"p"},"max_num_pipelines_per_indexer")," parameter is only available for sources that can be distributed: Kafka, GCP PubSub and Pulsar(coming soon)."),(0,r.kt)("p",null,"The maximum number of indexing pipelines defines the limit of pipelines spawned for the source on a given indexer.\nThis maximum can be reached only if there are enough ",(0,r.kt)("inlineCode",{parentName:"p"},"desired_num_pipelines")," to run."),(0,r.kt)("admonition",{type:"note"},(0,r.kt)("p",{parentName:"admonition"},"With the following parameters, only one pipeline will run on one indexer."),(0,r.kt)("ul",{parentName:"admonition"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"max_num_pipelines_per_indexer=2")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"desired_num_pipelines=1")))),(0,r.kt)("h2",{id:"desired-number-of-pipelines"},"Desired number of pipelines"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"desired_num_pipelines")," parameter is only available for sources that can be distributed: Kafka, GCP PubSub and Pulsar (coming soon)."),(0,r.kt)("p",null,'The desired number of indexing pipelines defines the number of pipelines to run on a cluster for the source. It is a "desired"\nnumber as it cannot be reach it there is not enough indexers in\nthe cluster.'),(0,r.kt)("admonition",{type:"note"},(0,r.kt)("p",{parentName:"admonition"},"With the following parameters, only one pipeline will start on the sole indexer."),(0,r.kt)("ul",{parentName:"admonition"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"max_num_pipelines_per_indexer=1")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"desired_num_pipelines=2")))),(0,r.kt)("h2",{id:"transfor
1m-parameters"},"Transform parameters"),(0,r.kt)("p",null,"For all source types but the ",(0,r.kt)("inlineCode",{parentName:"p"},"ingest-api"),", ingested documents can be transformed before being indexed using ",(0,r.kt)("a",{parentName:"p",href:"https://vector.dev/docs/reference/vrl/"},"Vector Remap Language (VRL)")," scripts."),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Property"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Default value"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"script")),(0,r.kt)("td",{parentName:"tr",align:null},"Source code of the VRL program executed to transform documents."),(0,r.kt)("td",{parentName:"tr",align:null},"required")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"timezone")),(0,r.kt)("td",{parentName:"tr",align:null},"Timezone used in the VRL program for date and time manipulations. It must be a valid name in the ",(0,r.kt)("a",{parentName:"td",href:"https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"},"TZ database")),(0,r.kt)("td",{parentName:"tr",align:null},(0,r.kt)("inlineCode",{parentName:"td"},"UTC"))))),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"# Your source config here\n# ...\ntransform:\n script: |\n .message = downcase(string!(.message))\n .timestamp = now()\n del(.username)\n timezone: local\n")),(0,r.kt)("h2",{id:"input-format"},"Input format"),(0,r.kt)("p",null,"The ",(0,r.kt)("inlineCode",{parentName:"p"},"input_format")," parameter specifies the expected data format of the source. Two formats are currently supported:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"json"),": JSON, the default"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"plain_text"),": unstructured text document")),(0,r.kt)("p",null,"Internally, Quickwit can only index JSON data. To allow the ingestion of plain text documents, Quickwit transform them on the fly into JSON objects of the following form: ",(0,r.kt)("inlineCode",{parentName:"p"},'{"plain_text": "<original plain text document>"}'),". Then, they can be optionally transformed into more complex documents using a VRL script. (see ",(0,r.kt)("a",{parentName:"p",href:"#transform-parameters"},"transform feature"),")."),(0,r.kt)("p",null,"The following is an example of how one could parse and transform a CSV dataset containing a list of users described by 3 attributes: first name, last name, and age."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"# Your source config here\n# ...\ninput_format: plain_text\ntransform:\n script: |\n user = parse_csv!(.plain_text)\n .first_name = user[0]\n .last_name = user[1]\n .age = to_int!(user[2])\n del(.plain_text)\n")),(0,r.kt)("h2",{id:"enablingdisabling-a-source-from-an-index"},"Enabling/Disabling a source from an index"),(0,r.kt)("p",null,"A source can be enabled or disabled from an index using the ",(0,r.kt)("a",{parentName:"p",href:"/docs/0.7.1/reference/cli"},"CLI command")," ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit source enable")," or ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit source disable"),":"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"quickwit source disable --index my-index --source my-source\n")),(0,r.kt)("p",null,"A source is enabled by default. When disabling a source, the related indexing pipelines will be shut down on each relevant indexer and indexing for this source will be paused."),(0,r.kt)("h2",{id:"deleting-a-source-from-an-index"},"Deleting a source from an index"),(0,r.kt)("p",null,"A source can be removed from an index using the ",(0,r.kt)("a",{parentName:"p",href:"/docs/0.7.1/reference/cli"},"CLI command")," ",(0,r.kt)("inlineCode",{parentName:"p"},"quickwit source delete"),":"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-bash"},"quickwit source delete --index my-index --source my-source\n")),(0,r.kt)("p",null,"When deleting a source, the checkpoint associated with the source is also removed."))}d.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.