https://spiceai.org/assets/js/e47a533f.bdb117e7.js

1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([["109096"],{152729(e,s,n){n.r(s),n.d(s,{metadata:()=>r,default:()=>h,frontMatter:()=>i,contentTitle:()=>d,toc:()=>l,assets:()=>a});var r=JSON.parse('{"id":"components/data-connectors/elasticsearch/index","title":"Elasticsearch Data Connector","description":"Query Elasticsearch indexes as SQL tables in Spice, including kNN vector search, full-text search, and hybrid search.","source":"@site/versioned_docs/version-2.2.x/components/data-connectors/elasticsearch/index.md","sourceDirName":"components/data-connectors/elasticsearch","slug":"/components/data-connectors/elasticsearch/","permalink":"/docs/v2.2/components/data-connectors/elasticsearch/","draft":false,"unlisted":false,"editUrl":"https://github.com/spiceai/docs/edit/trunk/website/versioned_docs/version-2.2.x/components/data-connectors/elasticsearch/index.md","tags":[{"inline":false,"label":"Data Connectors","permalink":"/docs/v2.2/tags/data-connectors","description":"Data source connectors and integration patterns."},{"inline":false,"label":"Elasticsearch","permalink":"/docs/v2.2/tags/elasticsearch","description":"Elasticsearch data connector and vector engine integration."},{"inline":false,"label":"Search","permalink":"/docs/v2.2/tags/search","description":"Vector search, semantic search, and ranking capabilities."}],"version":"2.2.x","frontMatter":{"title":"Elasticsearch Data Connector","sidebar_label":"Elasticsearch Data Connector","description":"Query Elasticsearch indexes as SQL tables in Spice, including kNN vector search, full-text search, and hybrid search.","tags":["data-connectors","elasticsearch","search"]},"sidebar":"docs","previous":{"title":"Deployment Guide","permalink":"/docs/v2.2/components/data-connectors/dynamodb/deployment"},"next":{"title":"Deployment Guide","permalink":"/docs/v2.2/components/data-connectors/elasticsearch/deployment"}}'),c=n(474848),t=n(28453);let i={title:"Elasticsearch Data Connector",sidebar_label:"Elasticsearch Data Connector",description:"Query Elasticsearch indexes as SQL tables in Spice, including kNN vector search, full-text search, and hybrid search.",tags:["data-connectors","elasticsearch","search"]},d,a={},l=[{value:"Configuration",id:"configuration",level:2},{value:"<code>from</code>

1",id:"from",level:3},{value:"<code>name</code>",id:"name",level:3},{value:"<code>params</code>",id:"params",level:3},{value:"Types",id:"types",level:2},{value:"Querying",id:"querying",level:2},{value:"Vector and Full-Text Search",id:"vector-and-full-text-search",level:3},{value:"Authentication",id:"authentication",level:2},{value:"Limitations",id:"limitations",level:2},{value:"Cookbook",id:"cookbook",level:2}];function o(e){let s={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",li:"li",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsx)(s.p,{children:"The Elasticsearch Data Connector exposes Elasticsearch indexes as SQL tables in Spice. Index mappings are translated to Arrow schemas so that documents can be queried with federated SQL alongside data from other connectors."}),"\n",(0,c.jsxs)(s.p,{children:["To run ",(0,c.jsx)(s.a,{href:"../../features/search",children:"vector, full-text, or hybrid search"})," (the ",(0,c.jsx)(s.code,{children:"vector_search"}),", ",(0,c.jsx)(s.code,{children:"text_search"}),", and ",(0,c.jsx)(s.code,{children:"rrf"})," UDTFs) against an Elasticsearch index, the dataset must additionally be configured for search \u2014 as an ",(0,c.jsx)(s.a,{href:"../vectors/elasticsearch",children:"Elasticsearch Vector Engine"})," with an embedding model for vector search, and/or with full-text search columns for ",(0,c.jsx)(s.code,{children:"text_search"}),". See ",(0,c.jsx)(s.a,{href:"#vector-and-full-text-search",children:"Vector and Full-Text Search"})," below. Registering an index through the data connector alone exposes it for federated SQL but does not make it searchable through those UDTFs."]}),"\n",(0,c.jsx)(s.pre,{children:(0,c.jsx)(s.code,{className:"language-yaml",children:"datasets:\n  - from: elasticsearch:products\n    name: products\n    params:\n      elasticsearch_endpoint: https://localhost:9200\n      elasticsearch_user: ${secrets:es_user}\n      elasticsearch_pass: ${secrets:es_pass}\n"})}),"\n",(0,c.jsx)(s.admonition,{title:"Enterprise edition",type:"note",children:(0,c.jsxs)(s.p,{children:["The Elasticsearch connector is available in the Spice ",(0,c.jsx)(s.a,{href:"https://docs.spice.ai/docs/enterprise/getting-started/distributions",children:"Enterprise edition"}),"."]})}),"\n",(0,c.jsx)(s.h2,{id:"configuration",children:"Configuration"}),"\n",(0,c.jsx)(s.h3,{id:"from",children:(0,c.jsx)(s.code,{children:"from"})}),"\n",(0,c.jsxs)(s.p,{children:["The ",(0,c.jsx)(s.code,{children:"from"})," field takes the form ",(0,c.jsx)(s.code,{children:"elasticsearch:{index_name}"})," where ",(0,c.jsx)(s.code,{children:"index_name"})," is the Elasticsearch index to query."]}),"\n",(0,c.jsx)(s.pre,{children:(0,c.jsx)(s.code,{className:"language-yaml",children:"datasets:\n  - from: elasticsearch:products\n    name: products\n"})}),"\n",(0,c.jsxs)(s.p,{children:["Dot-separated paths may be used to refer to nested fields in query results (e.g. ",(0,c.jsx)(s.code,{children:"address.city"}),"); the connector flattens object mappings into Arrow columns using that convention."]}),"\n",(0,c.jsx)(s.h3,{id:"name",children:(0,c.jsx)(s.code,{children:"name"})}),"\n",(0,c.jsxs)(s.p,{children:["The dataset name used as the table name within Spice. The dataset name cannot be a ",(0,c.jsx)(s.a,{href:"../../reference/spicepod/keywords",children:"reserved keyword"}),"."]}),"\n",(0,c.jsx)(s.h3,{id:"params",children:(0,c.jsx)(s.code,{children:"params"})}),"\n",(0,c.jsxs)(s.p,{children:["The Elasticsearch connector accepts the following ",(0,c.jsx)(s.code,{children:"params"}),". Use the ",(0,c.jsx)(s.a,{href:"../secret-stores",children:"secret replacement syntax"})," to load credentials from a secret store."]}),"\n",(0,c.jsxs)(s.table,{children:[(0,c.jsx)(s.thead,{children:(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.th,{children:"Parameter Name"}),(0,c.jsx)(s.th,{children:"Description"}),(0,c.jsx)(s.th,{children:"Required"}),(0,c.jsx)(s.th,{children:"Default"})]})}),(0,c.jsxs)(s.tbody,{children:[(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"elasticsearch_endpoint"})}),(0,c.jsxs)(s.td,{children:["Cluster URL (e.g., ",(0,c.jsx)(s.code,{children:"https://localhost:9200"}),")."]}),(0,c.jsx)(s.td,{children:"Yes"}),(0,c.jsx)(s.td,{children:"-"})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"elasticsearch_user"})}),(0,c.jsx)(s.td,{children:"Username for HTTP basic authentication."}),(0,c.jsx)(s.td,{children:"No"}),(0,c.jsx)(s.td,{children:"-"})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"elasticsearch_pass"})}),(0,c.jsx)(s.td,{children:"Password for HTTP basic authentication."}),(0,c.jsx)(s.td,{children:"No"}),(0,c.jsx)(s.td,{children:"-"})]})]})]}),"\n",(0,c.jsx)(s.h2,{id:"types",children:"Types"}),"\n",(0,c.jsxs)(s.p,{children:["The connector derives an Arrow schema from each index's mapping via ",(0,c.jsx)(s.code,{children:"GET /<index>/_mapping"}),". Elasticsearch field types map to Arrow as follows:"]}),"\n",(0,c.jsxs)(s.table,{children:[(0,c.jsx)(s.thead,{children:(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.th,{children:"Elasticsearch Field Type"}),(0,c.jsx)(s.th,{children:"Arrow Type"}),(0,c.jsx)(s.th,{children:"Notes"})]})}),(0,c.jsxs)(s.tbody,{children:[(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"text"}),", ",(0,c.jsx)(s.code,{children:"keyword"}),", ",(0,c.jsx)(s.code,{children:"wildcard"}),", ",(0,c.jsx)(s.code,{children:"constant_keyword"}),", ",(0,c.jsx)(s.code,{children:"match_only_text"})]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"long"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Int64"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"unsigned_long"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"UInt64"})}),(0,c.jsxs)(s.td,{children:["Accepts both numeric values and digit strings (JS clients commonly serialize values > 2",(0,c.jsx)("sup",{children:"53"}),"-1 as strings)."]})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"integer"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Int32"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"short"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Int16"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"byte"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Int8"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"double"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Float64"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"float"}),", ",(0,c.jsx)(s.code,{children:"half_float"}),", ",(0,c.jsx)(s.code,{children:"scaled_float"})]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Float32"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"boolean"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Boolean"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"date"}),", ",(0,c.jsx)(s.code,{children:"date_nanos"})]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{children:"ES dates are flexibly formatted; preserved as strings."})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"binary"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{children:"Base64-encoded in the JSON response."})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"ip"})}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"dense_vector"})," (with ",(0,c.jsx)(s.code,{children:"dims"}),")"]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"FixedSizeList<Float32, dims>"})}),(0,c.jsxs)(s.td,{children:["Required ",(0,c.jsx)(s.code,{children:"dims"})," field must fit in ",(0,c.jsx)(s.code,{children:"i32"}),"."]})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"dense_vector"})," (missing ",(0,c.jsx)(s.code,{children:"dims"}),")"]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{children:"Falls back to raw JSON when dims cannot be resolved."})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"object"})," (with sub-fields)"]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.em,{children:"(flattened)"})}),(0,c.jsxs)(s.td,{children:["Expanded into dot-separated columns (e.g. ",(0,c.jsx)(s.code,{children:"address.city"}),")."]})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsxs)(s.td,{children:[(0,c.jsx)(s.code,{children:"object"})," (no sub-fields), ",(0,c.jsx)(s.code,{children:"nested"})]}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{children:"Serialized JSON."})]}),(0,c.jsxs)(s.tr,{children:[(0,c.jsx)(s.td,{children:"Any other mapping type"}),(0,c.jsx)(s.td,{children:(0,c.jsx)(s.code,{children:"Utf8"})}),(0,c.jsx)(s.td,{children:"Fallback \u2014 the raw JSON value is preserved as a string."})]})]})]}),"\n",(0,c.jsxs)(s.p,{children:["Nested ",(0,c.jsx)(s.code,{children:"object"})," fields are flattened by concatenating field names with dots (e.g. ",(0,c.jsx)(s.code,{children:"address.city"}),"). ",(0,c.jsx)(s.code,{children:"nested"})," fields are preserved as JSON strings because per-document ordering must be retained."]}),"\n",(0,c.jsx)(s.h2,{id:"querying",children:"Querying"}),"\n",(0,c.jsx)(s.p,{children:"After registering a dataset, query it like any other Spice table:"}),"\n",(0,c.jsx)(s.pre,{children:(0,c.jsx)(s.code,{className:"language-sql",children:"SELECT name, price\nFROM products\nWHERE price > 100\nORDER BY price DESC\nLIMIT 10;\n"})}),"\n",(0,c.jsx)(s.h3,{id:"vector-and-full-text-search",children:"Vector and Full-Text Search"}),"\n",(0,c.jsxs)(s.p,{children:["An Elasticsearch dataset is ",(0,c.jsx)(s.strong,{children:"not"})," searchable through the search UDTFs by virtue of being registered with the data connector. To enable search against an Elasticsearch index, configure the dataset for search:"]}),"\n",(0,c.jsxs)(s.ul,{children:["\n",(0,c.jsxs)(s.li,{children:["For ",(0,c.jsx)(s.strong,{children:"vector"})," and ",(0,c.jsx)(s.strong,{children:"hybrid"})," search, configure the dataset as an ",(0,c.jsx)(s.a,{href:"../vectors/elasticsearch",children:"Elasticsearch Vector Engine"})," (",(0,c.jsx)(s.code,{children:"vectors: { engine: elasticsearch, enabled: true }"}),") with a column-level ",(0,c.jsx)(s.code,{children:"embeddings"})," entry naming an embedding model. The embedding model is required \u2014 it is used to embed the query text at search time."]}),"\n",(0,c.jsxs)(s.li,{children:["For ",(0,c.jsx)(s.strong,{children:"full-text"})," search, enable ",(0,c.jsx)(s.code,{children:"full_text_search"})," on the column(s) to search."]}),"\n"]}),"\n",(0,c.jsx)(s.p,{children:"Once configured, the following UDTFs are available against the dataset:"}),"\n",(0,c.jsxs)(s.ul,{children:["\n",(0,c.jsxs)(s.li,{children:[(0,c.jsx)(s.strong,{children:"Vector similarity search"})," via ",(0,c.jsx)(s.a,{href:"../../reference/sql/search#vector-search-vector_search",children:(0,c.jsx)(s.code,{children:"vector_search"})})," \u2014 executed natively as an Elasticsearch kNN query."]}),"\n",(0,c.jsxs)(s.li,{children:[(0,c.jsx)(s.strong,{children:"Full-text search"})," via ",(0,c.jsx)(s.a,{href:"../../reference/sql/search#full-text-search-text_search",children:(0,c.jsx)(s.code,{children:"text_search"})})," \u2014 executed using Elasticsearch's native BM25 ranking."]}),"\n",(0,c.jsxs)(s.li,{children:[(0,c.jsx)(s.strong,{children:"Hybrid search"})," via ",(0,c.jsx)(s.a,{href:"../../reference/sql/search#reciprocal-rank-fusion-rrf",children:(0,c.jsx)(s.code,{children:"rrf"})}

1)," \u2014 combining both with Reciprocal Rank Fusion."]}),"\n"]}),"\n",(0,c.jsx)(s.p,{children:"These operations run against the Elasticsearch cluster directly rather than ingesting vectors into an accelerator, keeping indexing and search colocated in Elasticsearch."}),"\n",(0,c.jsx)(s.p,{children:"Example:"}),"\n",(0,c.jsx)(s.pre,{children:(0,c.jsx)(s.code,{className:"language-sql",children:"-- kNN vector search against Elasticsearch\nSELECT product_id, name, score\nFROM vector_search(products, 'wireless noise cancelling headphones')\nORDER BY score DESC\nLIMIT 10;\n\n-- BM25 full-text search\nSELECT product_id, name, score\nFROM text_search(products, 'headphones waterproof', description)\nORDER BY score DESC\nLIMIT 10;\n\n-- Hybrid search via RRF\nSELECT product_id, name, fused_score\nFROM rrf(\n  vector_search(products, 'wireless noise cancelling headphones'),\n  text_search(products, 'headphones waterproof', description),\n  join_key => 'product_id'\n)\nORDER BY fused_score DESC\nLIMIT 10;\n"})}),"\n",(0,c.jsxs)(s.p,{children:["See ",(0,c.jsx)(s.a,{href:"../../features/search",children:"Search Functionality"})," for the full search feature guide."]}),"\n",(0,c.jsx)(s.h2,{id:"authentication",children:"Authentication"}),"\n",(0,c.jsxs)(s.p,{children:["The connector uses HTTP basic authentication when ",(0,c.jsx)(s.code,{children:"elasticsearch_user"})," and ",(0,c.jsx)(s.code,{children:"elasticsearch_pass"})," are provided. For production deployments, store credentials in a ",(0,c.jsx)(s.a,{href:"../secret-stores",children:"secret store"})," and reference them with ",(0,c.jsx)(s.code,{children:"${secrets:...}"})," rather than hard-coding them in ",(0,c.jsx)(s.code,{children:"spicepod.yaml"}),"."]}),"\n",(0,c.jsxs)(s.p,{children:["TLS is enabled automatically for ",(0,c.jsx)(s.code,{children:"https://"})," endpoints."]}),"\n",(0,c.jsx)(s.h2,{id:"limitations",children:"Limitations"}),"\n",(0,c.jsxs)(s.ul,{children:["\n",(0,c.jsx)(s.li,{children:"Nested object fields are exposed as JSON strings rather than structured columns."}),"\n",(0,c.jsxs)(s.li,{children:[(0,c.jsx)(s.code,{children:"date"})," and ",(0,c.jsx)(s.code,{children:"date_nanos"})," fields are preserved as strings because Elasticsearch accepts heterogeneous date formats; cast to a timestamp in SQL when numeric comparison is required."]}),"\n",(0,c.jsxs)(s.li,{children:[(0,c.jsx)(s.code,{children:"dense_vector"})," fields without a declared ",(0,c.jsx)(s.code,{children:"dims"})," value fall back to ",(0,c.jsx)(s.code,{children:"Utf8"})," and are not usable as a vector column."]}),"\n",(0,c.jsxs)(s.li,{children:["For queries with ",(0,c.jsx)(s.code,{children:"LIMIT N"})," where N \u2264 10,000, the connector issues a single ",(0,c.jsx)(s.code,{children:"_search"})," request. For larger result sets or queries without ",(0,c.jsx)(s.code,{children:"LIMIT"}),", the connector automatically paginates using Point-In-Time (PIT) + ",(0,c.jsx)(s.code,{children:"search_after"}),", fetching all matching documents in 10,000-hit batches."]}),"\n",(0,c.jsxs)(s.li,{children:["SQL ",(0,c.jsx)(s.code,{children:"WHERE"})," predicates are not pushed down to the Elasticsearch query DSL; all filter expressions are evaluated locally by DataFusion after fetching results (only ",(0,c.jsx)(s.code,{children:"LIMIT"})," is pushed down, as the query ",(0,c.jsx)(s.code,{children:"size"}),")."]}),"\n"]}

1),"\n",(0,c.jsxs)(s.p,{children:["Elasticsearch can also be configured as a ",(0,c.jsx)(s.a,{href:"../vectors/elasticsearch",children:"Vector Engine"})," for datasets sourced from other connectors (storing Spice-managed embeddings in Elasticsearch rather than querying an existing index)."]}),"\n",(0,c.jsx)(s.h2,{id:"cookbook",children:"Cookbook"}),"\n",(0,c.jsxs)(s.ul,{children:["\n",(0,c.jsxs)(s.li,{children:["A cookbook recipe to configure Elasticsearch as a data connector in Spice. ",(0,c.jsx)(s.a,{href:"https://github.com/spiceai/cookbook/tree/trunk/elasticsearch/connector#readme",children:"Elasticsearch Data Connector"})]}),"\n"]})]})}function h(e={}){let{wrapper:s}={...(0,t.R)(),...e.components};return s?(0,c.jsx)(s,{...e,children:(0,c.jsx)(o,{...e})}):o(e)}},28453(e,s,n){n.d(s,{R:()=>i,x:()=>d});var r=n(296540);let c={},t=r.createContext(c);function i(e){let s=r.useContext(t);return r.useMemo(function(){return"function"==typeof e?e(s):{...s,...e}},[s,e])}function d(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(c):e.components||c:i(e.components),r.createElement(t.Provider,{value:s},e.children)}}}]);