1"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[3838],{39530:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>a,default:()=>p,frontMatter:()=>s,metadata:()=>i,toc:()=>l});const i=JSON.parse('{"id":"plugins/connector-requirements","title":"Requirements for Openk9 connector","description":"Check existing Openk9 connector to search for connector based on your data source. Explore connectors section.","source":"@site/docs/plugins/connectors-requirements.md","sourceDirName":"plugins","slug":"/plugins/connector-requirements","permalink":"/docs/plugins/connector-requirements","draft":false,"unlisted":false,"editUrl":"https://github.com/smclab/openk9/tree/main/docs/docs/plugins/connectors-requirements.md","tags":[],"version":"current","frontMatter":{"id":"connector-requirements","title":"Requirements for Openk9 connector"},"sidebar":"sidebar","previous":{"title":"Create tabs","permalink":"/docs/configuration/create-tabs"},"next":{"title":"Archetype Project for connector","permalink":"/docs/plugins/archetype-project"}}');var o=t(74848),r=t(28453);const s={id:"connector-requirements",title:"Requirements for Openk9 connector"},a=void 0,c={},l=[{value:"What is an Openk9 connector",id:"what-is-an-openk9-connector",level:2},{value:"Prerequisites",id:"prerequisites",level:2},{value:"Connector Trigger Endpoint",id:"connector-trigger-endpoint",level:3},{value:"Openk9 Ingestion Interface",id:"openk9-ingestion-interface",level:3},{value:"Datasource Payload",id:"datasource-payload",level:4},{value:"Resources",id:"resources",level:4},{value:"Acl",id:"acl",level:4},{value:"Type",id:"type",level:4},{value:"Last",id:"last",level:4},{value:"Asynchronous implementation of data extraction",id:"asynchronous-implementation-of-data-extraction",level:3},{value:"Management endpoints",id:"management-endpoints",level:3},{value:"Openk9 Health Check Endpoint",id:"openk9-health-check-endpoint",level:4},{value:"Openk9 Form Endpoint",id:"openk9-form-endpoint",level:4},{value:"Openk9 Sample Endpoint",id:"openk9-sample-endpoint",level:4}];function d(e){const n={a:"a",blockquote:"blockquote",br:"br",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,r.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(n.p,{children:["Check existing Openk9 connector to search for connector based on your data source. Explore ",(0,o.jsx)(n.a,{href:"/plugins",children:"connectors section"}),"."]}),"\n",(0,o.jsx)(n.p,{children:"If not available, to ingest data from your data source you need to realize new Openk9 connector."}),"\n",(0,o.jsxs)(n.p,{children:["Samples to start for realize new connector are available in ",(0,o.jsx)(n.a,{href:"python-base-connector.md",children:"Python"})," and ",(0,o.jsx)(n.a,{href:"python-java-connector.md",children:"Java"})]}),"\n",(0,o.jsx)(n.h2,{id:"what-is-an-openk9-connector",children:"What is an Openk9 connector"}),"\n",(0,o.jsxs)(n.p,{children:["A connector in Openk9 is a service that get data from external data source and send these to ",(0,o.jsx)(n.a,{href:"/docs/ingestion",children:"Openk9 Ingestion component"}),"."]}),"\n",(0,o.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,o.jsx)(n.p,{children:"No prerequisites are required about programming language. You can realize it using your favorite programming language."}),"\n",(0,o.jsx)(n.p,{children:"There are some specifications you need to respect when develop new Openk9 Connector."}),"\n",(0,o.jsx)(n.h3,{id:"connector-trigger-endpoint",children:"Connector Trigger Endpoint"}),"\n",(0,o.jsx)(n.p,{children:"An Openk9 connector must expose an endpoint which is then used by Openk9 to call the connector itself."}),"\n",(0,o.jsx)(n.p,{children:"This endpoint must have following mandatory parameters:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"datasourceId"}),": Id of Datasource"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"scheduleId"}),": Id of Schedule"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"tenantId"}),": Id of Tenant"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"timestamp"}),": Timestamp relative to last ingestion date"]}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"The different ids coming from Openk9 then are propagated again to Openk9 Ingestion interface. They are used to properly forward data inside Openk9 processing steps."}),"\n",(0,o.jsxs)(n.p,{children:["The field ",(0,o.jsx)(n.strong,{children:"timestamp"})," can be used by connector to filter data and get only new or modified data."]}),"\n",(0,o.jsx)(n.h3,{id:"openk9-ingestion-interface",children:"Openk9 Ingestion Interface"}),"\n",(0,o.jsxs)(n.p,{children:["Connector when send data to Openk9 needs to respect ",(0,o.jsx)(n.a,{href:"/",children:"ingestion Api interface"})," and pass in request body following parameters:"]}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"contentId"}),": unique contentId for data sent to Openk9"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"datasourceId"}),": id of Datasource (received in request and propagated for)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"scheduleId"}),": id of Schedule (received in request and propagated for)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"tenantId"}),": id of Tenant (received in request and propagated for)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"parsingDate"}),": date when data extraction starts"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"rawContent"}),": raw text content of data"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"datasourcePayload"}),": structured content of data"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"resources"}),": resources attached to data (Optional)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"acl"}),": acl associated with data (Optional)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"type"}),": used to specify the type of message. (Optional)"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"last"}),": boolean to indicates all data are sent to Openk9. (Optional)"]}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"datasource-payload",children:"Datasource Payload"}),"\n",(0,o.jsx)(n.p,{children:"Datasource payload consists in a structured object where you can pass metadata of the document you need to send to Openk9."}),"\n",(0,o.jsx)(n.p,{children:"For example in casa of a web page datasource payload could be like following:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n "web": {\n "title": "Lorem ipsum Wikipendia",\n "content": "Lorem ipsum....",\n "url": "https://it.wikipedia.org/wiki/Lorem_ipsum"\n }\n}\n'})}),"\n",(0,o.jsx)(n.p,{children:"You can add all metadata you want, and map them to different key inside the object."}),"\n",(0,o.jsx)(n.h4,{id:"resources",children:"Resources"}),"\n",(0,o.jsxs)(n.p,{children:["Using parameter ",(0,o.jsx)(n.em,{children:"resources"})," is possibile to pass to Openk9 binary resources. To pass binaries to Openk9 you need to encode them to base64 string and then\npass to parameters resources, building object like following:"]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n {\n "binaries": [\n {\n "data": "<base64-string>",\n "contentType": "application/pdf",\n "id": 1,\n "name": "example.pdf"\n },\n {\n "data": "<base64-string>",\n "contentType": "image/png",\n "id": 2,\n "name": "image.png"\n }\n ]\n }\n}\n'})}),"\n",(0,o.jsx)(n.p,{children:"When Openk9 get message, extracts binaries from message and stored them to S3 Minio storage. Then binaries are available inside Openk9 System to perform any sort of processing."}),"\n",(0,o.jsx)(n.p,{children:"Validating this parameter is not mandatory. If you don't set it, Openk9 interprets message haven't any binaries associated."}),"\n",(0,o.jsx)(n.h4,{id:"acl",children:"Acl"}),"\n",(0,o.jsxs)(n.p,{children:["Using ",(0,o.jsx)(n.em,{children:"acl"})," parameter is possible to index, together with document, the related acl (Access Control List).\nTo pass acl you need first to be able to get them from original data. Then you need to construct an object like following:"]}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n {\n "rolesName": ["site_admin", "site_editor", "site_viewer"],\n }\n}\n'})}),"\n",(0,o.jsx)(n.p,{children:"Validating this parameter is not mandatory. If you don't set it, Openk9 interprets the data like a data without any acl and consequentially it results visible to all."}),"\n",(0,o.jsx)(n.h4,{id:"type",children:"Type"}),"\n",(0,o.jsxs)(n.p,{children:["Using ",(0,o.jsx)(n.em,{children:"type"})," parameter is possibile to label message for its type.\nThree different types of types are supported:"]}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"DOCUMENT"}),": is default type. Identify a message with document to be indexed"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"HALT"}),": HALT type identify a message that report to Openk9 to stop running schedulation. This can be used when an error occurs during data extraction and the expected behaviour is to stop extraction and stop also the correspondign schedulation inside Openk9"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"LAST"}),": to report that data extraction is finished"]}),"\n"]}),"\n",(0,o.jsx)(n.h4,{id:"last",children:"Last"}),"\n",(0,o.jsx)(n.p,{children:"Using this parameter it is possibile to report that data extraction is finished.\nIts behaviour is the same as Type parameter with value LAST."}),"\n",(0,o.jsxs)(n.blockquote,{children:["\n",(0,o.jsxs)(n.p,{children:[(0,o.jsx)(n.strong,{children:"Note"}),(0,o.jsx)(n.br,{}),"\n",(0,o.jsx)(n.code,{children:"parsingDate"})," is mandatory when the ",(0,o.jsx)(n.strong,{children:"Type"})," parameter is set to LAST or when using the ",(0,o.jsx)(n.strong,{children:"Last"})," parameter."]}),"\n"]}),"\n",(0,o.jsx)(n.h3,{id:"asynchronous-implementation-of-data-extraction",children:"Asynchronous implementation of data extraction"}),"\n",(0,o.jsx)(n.p,{children:"When Openk9 calls the connector's trigger endpoint, the connector must start the extraction asynchronously on a
1separate thread and possibly respond to Openk9 if this started correctly or if instead there was some error (it sends an exception with a code error and related message).\nIf Openk9 starts correctly at that point it disconnects, starts a schedule and waits for messages from the connector."}),"\n",(0,o.jsx)(n.p,{children:"The connector begins data extraction and sends this data one by one to the above ingestion endpoint."}),"\n",(0,o.jsx)(n.p,{children:"At this point two cases can occur:"}),"\n",(0,o.jsxs)(n.ol,{children:["\n",(0,o.jsx)(n.li,{children:"The connector correctly extracts all documents from the source and sends them all to Openk9 wconsequentiallyithout errors."}),"\n",(0,o.jsx)(n.li,{children:"The connector encounters an error during data extraction (for example, it does not correctly extract a document from the source or breaks when it sends it to ingestion)"}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"In case number 1, the connector, after sending all the documents, sends a message with last a true."}),"\n",(0,o.jsx)(n.p,{children:"In case number 2 the connector can:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:"ignore the errors that occur and get to the end by sending the last"}),"\n",(0,o.jsx)(n.li,{children:"in case of error, stop data extraction and signal Openk9 to stop scheduling"}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"To signal Openk9 to interrupt a schedule, a message with type HALT must be sent to the ingestion."}),"\n",(0,o.jsx)(n.h3,{id:"management-endpoints",children:"Management endpoints"}),"\n",(0,o.jsx)(n.p,{children:"Connector can expose some management endpoints needed to configure and connect in proper way these to Openk9 instance."}),"\n",(0,o.jsx)(n.p,{children:"Every following endpoint is not mandatory to connect and use connector. But if implemented, configuration through admin ui is simpler."}),"\n",(0,o.jsx)(n.h4,{id:"openk9-health-check-endpoint",children:"Openk9 Health Check Endpoint"}),"\n",(0,o.jsx)(n.p,{children:"When you connect and configure your own connector in Openk9 admin Ui, you can perform health check directly from\nadmin interface. In this way, you can check if your connector is healthy.\nTo enable this feature your own connector needs to implement health check endpoint respecting specification as follows:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"curl -X 'GET' \\\n'http://localhost:32805/health' \\\n-H 'accept: application/json'\n"})}),"\n",(0,o.jsx)(n.p,{children:"The endpoint return JSON response like this:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n "status": "UP/DOWN",\n "checks": [\n {\n "name": "OpenSearch cluster health check",\n "status": "UP",\n "data": {\n "status": "Green"\n }\n },\n {\n "name": "SmallRye Reactive Messaging - startup check",\n "status": "UP"\n }\n ]\n}\n'})}),"\n",(0,o.jsxs)(n.p,{children:[(0,o.jsx)(n.em,{children:"Status"})," parameter is mandatory in the response. You can then add other parameters, like for example details, but they are not analyzed by Openk9 to perform health check. Openk9 uses only ",(0,o.jsx)(n.em,{children:"status"})," parameter for this scope."]}),"\n",(0,o.jsx)(n.h4,{id:"openk9-form-endpoint",children:"Openk9 Form Endpoint"}),"\n",(0,o.jsx)(n.p,{children:"When you connect and configure your own connector in Openk9 admin Ui, Openk9 can create a configuration form with specific\ndata for your own connector in automatic way.\nTo enable this feature your own connector needs to implement form endpoint respecting specification as follows:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"curl -X 'GET' \\\n 'http://localhost:32805/form' \\\n -H 'accept: application/json'\n"})}),"\n",(0,o.jsx)(n.p,{children:"The endpoint return JSON response like this:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n "fields": [\n {\n "label": "Sitemap Urls",\n "name": "sitemapUrls",\n "field": "sitemapUrls",\n "type": "list",\n "size": 4,\n "required": true,\n "values": [\n {\n "value": "sitemapUrl1",\n "label": "sitemapUrl1",\n "isDefault": false\n },\n {\n "value": "sitemapUrl2",\n "label": "sitemapUrl2",\n "isDefault": false\n }\n ],\n "info": "",\n "placeholder": "string",\n "validator": {\n "min": 0,\n "max": 100,\n "regex": "/[[:alnum:]]+/"\n }\n },\n {\n "label": "Allowed Domains",\n "name": "allowedDomains",\n "type": "list",\n "size": 4,\n "required": false,\n "values": [],\n "info": "",\n "validator": {\n "min": 0,\n "max": 100,\n "regex": "/[[:alnum:]]+/"\n }\n },\n {\n "label": "Body Tag",\n "name": "bodyTag",\n "type": "text",\n "size": 4,\n "required": true,\n "values": [\n {\n "value": "body",\n "isDefault": true\n }\n ],\n "info": "",\n "validator": {\n "min": 0,\n "max": 100,\n "regex": "/[[:alnum:]]+/"\n }\n },\n {\n "label": "Title tag",\n "name": "titleTag",\n "type": "text",\n "size": 4,\n "required": true,\n "info": "",\n "values": [\n {\n "value": "title::text",\n "isDefault": true\n }\n ],\n "validator": {\n "min": 0,\n "max": 100,\n "regex": "/[[:alnum:]]+/"\n }\n },\n {\n "label": "Page Count",\n "name": "pageCount",\n "type": "number",\n "size": 4,\n "required": true,\n "info": "",\n "placeholder: "",\n "values": [\n {\n "value": "0",\n "isDefault": true\n }\n ],\n "validator": {\n "min": 0,\n "max": 100,\n "regex": "/[[:alnum:]]+/"\n }\n }\n ]\n}\n'})}),"\n",(0,o.jsxs)(n.p,{children:["The response must return an object within an array ",(0,o.jsx)(n.em,{children:"fields"}),"."]}),"\n",(0,o.jsx)(n.p,{children:"Objects inside array fields must have following parameters:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"label"}),": label associated with input in form"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"name"}),": name associated to field inside form"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"type"}),": type used to render correctly field in form. Is possible to choose between: text, number, list, select, boolean"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"size"}),": size of input in form"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"required"}),": if is mandatory or not"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"info"}),": info to rendere in information tip in form"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"values"})," default values to rendere in input"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"placeholder"}),": placeholder to render in input in form"]}),"\n",(0,o.jsxs)(n.li,{children:[(0,o.jsx)(n.strong,{children:"val
1idator"}),": type of validation to apply in input in form"]}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"Based on response Openk9 admin Ui can build a correct form to configure your own connector when is connected."}),"\n",(0,o.jsx)(n.h4,{id:"openk9-sample-endpoint",children:"Openk9 Sample Endpoint"}),"\n",(0,o.jsx)(n.p,{children:"When you connect and configure your own connector in Openk9 admin Ui, openk9 enable feature to generate index mapping\nfrom a document sample taken from this endpoint's response.\nTo enable this feature your own connector needs to implement sample endpoint respecting specification as follows:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:"curl -X 'GET' \\\n 'http://localhost:32805/sample' \\\n -H 'accept: application/json'\n"})}),"\n",(0,o.jsxs)(n.p,{children:["This endpoint must return a JSON response respecting Ingestion Openk9 Request. In particular mapping is generated analyzing\ninformation inside ",(0,o.jsx)(n.em,{children:"datasource payload"}),"."]}),"\n",(0,o.jsx)(n.p,{children:"In case, for example, of a web crawler a response could be:"}),"\n",(0,o.jsx)(n.pre,{children:(0,o.jsx)(n.code,{children:'{\n "parsingDate": 1705326198961,\n "datasourceId": "1",\n "contentId": 3062252989527342,\n "rawContent": "Lorem ipsum \xe8 un testo segnaposto utilizzato da grafici, progettisti, programmatori e tipografi a modo riempitivo per bozzetti e prove grafiche.[1] \xc8 un testo privo di senso, composto da parole (o parti di parole) in lingua latina, riprese pseudocasualmente dal De finibus bonorum et malorum scritto da Cicerone del 45 a.C, a volte alterate con l\'inserzione di passaggi ironici. La caratteristica principale \xe8 data dal fatto che offre una distribuzione delle lettere uniforme, apparendo come un normale blocco di testo leggibile.",\n "datasourcePayload": {\n "web": {\n "url": "https://it.wikipedia.org/wiki/Lorem_ipsum",\n "content": "Lorem ipsum \xe8 un testo segnaposto utilizzato da grafici, progettisti, programmatori e tipografi a modo riempitivo per bozzetti e prove grafiche.[1] \xc8 un testo privo di senso, composto da parole (o parti di parole) in lingua latina, riprese pseudocasualmente dal De finibus bonorum et malorum scritto da Cicerone del 45 a.C, a volte alterate con l\'inserzione di passaggi ironici. La caratteristica principale \xe8 data dal fatto che offre una distribuzione delle lettere uniforme, apparendo come un normale blocco di testo leggibile.",\n "title": "Lorem ipsum - Wikipedia",\n "favicon": "https://it.wikipedia.org/favicon.ico"\n }Navigation Skip to Content Navigation Menu Esigenze Close menu Esigenze Risolviamo problemi complessi scegliendo le soluzioni pi\xc3\xb9 adatte a ciascun cliente Gestione processi e dati Migliorare i processi produttivi Gestiamo la produzione dal collegamento\n },\n "resources": {\n "binaries": []\n },\n "scheduleId": "1"\n}\n'})}),"\n",(0,o.jsx)(n.p,{children:"This is exactly an example of message that connector construct and send to Openk9 during data extraction. Openk9 can use it to inherits structure of data\nand create correct mapping for indexes."})]})}function p(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},28453:(e,n,t)=>{t.d(n,{R:()=>s,x:()=>a});var i=t(96540);const o={},r=i.createContext(o);function s(e){const n=i.useContext(r);return i.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),i.createElement(r.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.