1"use strict";(globalThis.webpackChunkopenrag_docs=globalThis.webpackChunkopenrag_docs||[]).push([[2410],{4013(e,n,a){a.r(n),a.d(n,{assets:()=>c,contentTitle:()=>o,default:()=>l,frontMatter:()=>i,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"reference/observability","title":"Monitor OpenRAG with IBM Instana","description":"OpenRAG Backend can be instrumented with IBM Instana for application performance monitoring (APM) and distributed tracing.","source":"@site/docs/reference/observability.mdx","sourceDirName":"reference","slug":"/reference/observability","permalink":"/reference/observability","draft":false,"unlisted":false,"editUrl":"https://github.com/openrag/openrag/tree/main/docs/docs/reference/observability.mdx","tags":[],"version":"current","frontMatter":{"title":"Monitor OpenRAG with IBM Instana","slug":"/reference/observability"},"sidebar":"tutorialSidebar","previous":{"title":"Environment variables","permalink":"/reference/configuration"},"next":{"title":"SDK and MCP API references","permalink":"/reference/api-sdk-overview"}}');var r=a(4848),s=a(8453);const i={title:"Monitor OpenRAG with IBM Instana",slug:"/reference/observability"},o=void 0,c={},d=[{value:"How traces reach Instana",id:"how-traces-reach-instana",level:2},{value:"What gets exported",id:"what-gets-exported",level:2},{value:"Redact query parameters",id:"redact-query-parameters",level:3},{value:"Before you begin",id:"before-you-begin",level:2},{value:"Install the tracer",id:"install-the-tracer",level:3},{value:"Enable tracing in a self-managed deployment",id:"enable-tracing",level:2},{value:"Run the local host agent",id:"local-host-agent",level:2},{value:"Enable tracing in Kubernetes",id:"kubernetes",level:2},{value:"Enable tracing with the OpenRAG operator",id:"operator",level:3},{value:"Configure the tracer's own logging",id:"tracer-logging",level:2},{value:"Performance guardrails",id:"performance-guardrails",level:2},{value:"Logging spans",id:"logging-spans",level:3},{value:"Stack traces on exit spans",id:"exit-span-stack-traces",level:3},{value:"Environment variable reference",id:"variables",level:2}];function h(e){const n={a:"a",admonition:"admonition",code:"code",em:"em",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(n.p,{children:["OpenRAG Backend can be instrumented with ",(0,r.jsx)(n.a,{href:"https://www.ibm.com/products/instana",children:"IBM Instana"})," for application performance monitoring (APM) and distributed tracing."]}),"\n",(0,r.jsxs)(n.p,{children:["Instrumentation is off by default. When you set ",(0,r.jsx)(n.code,{children:"INSTANA_ENABLED=true"}),", the backend loads Instana's Python tracer at startup, before the libraries the tracer instruments, and traces are collected automatically for:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Incoming API requests to the backend (FastAPI and Starlette)."}),"\n",(0,r.jsx)(n.li,{children:"Outgoing HTTP calls to Langflow, Docling, and model providers."}),"\n",(0,r.jsx)(n.li,{children:"OpenSearch queries."}),"\n",(0,r.jsx)(n.li,{children:"Database queries issued through SQLAlchemy, for both the default SQLite database and PostgreSQL."}),"\n",(0,r.jsxs)(n.li,{children:["Backend log records, when logging spans are enabled \u2014 off by default via ",(0,r.jsx)(n.code,{children:"INSTANA_TRACING_DISABLE=logging"})," (see ",(0,r.jsx)(n.a,{href:"#logging-spans",children:"Logging spans"}),")."]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"Only OpenRAG Backend is instrumented. Langflow, the frontend, OpenSearch, and Docling Serve aren't traced by this feature. Instana's host agent reports infrastructure metrics for those containers separately."}),"\n",(0,r.jsx)(n.h2,{id:"how-traces-reach-instana",children:"How traces reach Instana"}),"\n",(0,r.jsxs)(n.p,{children:["The tracer doesn't send data to your Instana tenant directly.\nIt sends spans to an Instana ",(0,r.jsx)(n.em,{children:"host agent"})," on port ",(0,r.jsx)(n.code,{children:"42699"}),", and the agent forwards them to your tenant's ingress endpoint:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-text",children:"OpenRAG Backend \u2500\u2500\u25b6 Instana host agent \u2500\u2500\u25b6 ingress-REGION-saas.instana.io:443\n (tracer) (:42699) (your Instana tenant)\n"})}),"\n",(0,r.jsxs)(n.p,{children:["Enabling APM therefore has two parts: configure the backend, and make sure an agent is reachable at ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_HOST:INSTANA_AGENT_PORT"}),".\nThe correct agent address depends on where the backend process runs:"]}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Deployment"}),(0,r.jsx)(n.th,{children:"Where the agent runs"}),(0,r.jsx)(n.th,{children:(0,r.jsx)(n.code,{children:"INSTANA_AGENT_HOST"})})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.a,{href:"/docker",children:"Self-managed containers"})}),(0,r.jsxs)(n.td,{children:["On the Docker host, started by ",(0,r.jsx)(n.code,{children:"make instana-agent-up"})]}),(0,r.jsxs)(n.td,{children:[(0,r.jsx)(n.code,{children:"host.docker.internal"}),", the Docker Compose default"]})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Backend on the host, such as ",(0,r.jsx)(n.code,{children:"make backend"})," for ",(0,r.jsx)(n.a,{href:"/support/contribute",children:"contributors"})]}),(0,r.jsx)(n.td,{children:"On the same host"}),(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"127.0.0.1"})})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Kubernetes"}),(0,r.jsxs)(n.td,{children:["An ",(0,r.jsx)(n.code,{children:"instana-agent"})," DaemonSet, one agent per node behind host port ",(0,r.jsx)(n.code,{children:"42699"})]}),(0,r.jsx)(n.td,{children:"The pod's own node IP, injected automatically"})]})]})]}),"\n",(0,r.jsx)(n.h2,{id:"what-gets-exported",children:"What gets exp
1orted"}),"\n",(0,r.jsx)(n.p,{children:"Trace data leaves your deployment: the tracer sends it to the host agent, and the agent forwards it to your Instana tenant.\nBefore you enable APM against a real corpus, it's worth knowing what a span carries."}),"\n",(0,r.jsxs)(n.p,{children:["Spans ",(0,r.jsx)(n.strong,{children:"don't"})," carry request or response bodies.\nChat prompts, retrieved chunk text, and uploaded document content are sent to the backend in request bodies, so they stay inside your deployment."]}),"\n",(0,r.jsxs)(n.p,{children:["Spans ",(0,r.jsx)(n.strong,{children:"do"})," carry:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["The request path and method, such as ",(0,r.jsx)(n.code,{children:"POST /v1/chat"}),", and the query string."]}),"\n",(0,r.jsx)(n.li,{children:"Outgoing request URLs for calls to Langflow, Docling, model providers, and OpenSearch."}),"\n",(0,r.jsx)(n.li,{children:"SQL statement text for database queries. Statements are parameterized, so bound values aren't included, and database credentials are stripped from the connection URL."}),"\n",(0,r.jsxs)(n.li,{children:["Log messages at ",(0,r.jsx)(n.code,{children:"WARNING"})," level and above, when logging spans are enabled \u2014 off by default (see ",(0,r.jsx)(n.a,{href:"#logging-spans",children:"Logging spans"}),"). A warning or an error that interpolates a filename, a user query, or chunk text exports that text."]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["The query string is the surface to pay attention to, because a few OpenRAG endpoints take free text there.\n",(0,r.jsx)(n.code,{children:"GET /v2/files/search?q=..."})," carries a user's search terms, and file listing takes ",(0,r.jsx)(n.code,{children:"search"})," and ",(0,r.jsx)(n.code,{children:"filename"})," parameters."]}),"\n",(0,r.jsx)(n.h3,{id:"redact-query-parameters",children:"Redact query parameters"}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_SECRETS"})," controls which query parameters the tracer replaces with ",(0,r.jsx)(n.code,{children:"<redacted>"})," before a span is sent. Its format is ",(0,r.jsx)(n.code,{children:"<matcher>:<name>[,<name>]"}),", where the matcher is one of ",(0,r.jsx)(n.code,{children:"equals"}),", ",(0,r.jsx)(n.code,{children:"equals-ignore-case"}),", ",(0,r.jsx)(n.code,{children:"contains"}),", ",(0,r.jsx)(n.code,{children:"contains-ignore-case"}),", or ",(0,r.jsx)(n.code,{children:"regex"}),"."]}),"\n",(0,r.jsxs)(n.p,{children:["The tracer's own default is ",(0,r.jsx)(n.code,{children:"contains-ignore-case:key,pass,secret"}),", which covers credential-shaped parameter names such as ",(0,r.jsx)(n.code,{children:"api_key"})," but leaves free-text parameters untouched \u2014 including the ",(0,r.jsx)(n.code,{children:"q"}),", ",(0,r.jsx)(n.code,{children:"search"}),", and ",(0,r.jsx)(n.code,{children:"filename"})," parameters above. OpenRAG ships a wider value instead, so this redaction applies out of the box; it's set for you in ",(0,r.jsx)(n.code,{children:".env.example"}),", ",(0,r.jsx)(n.code,{children:"docker-compose.yml"}),", the Helm chart, and the operator:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"INSTANA_SECRETS=regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\\Z,search\\Z,filename\\Z\n"})}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"q"}),", ",(0,r.jsx)(n.code,{children:"search"}),", and ",(0,r.jsx)(n.code,{children:"filename"})," are anchored with ",(0,r.jsx)(n.code,{children:"\\Z"})," (end-of-string) rather\nthan ",(0,r.jsx)(n.code,{children:"$"}),": the tracer matches with Python's ",(0,r.jsx)(n.code,{children:"re.match()"}),", which already\nanchors at the start of the string, and ",(0,r.jsx)(n.code,{children:".env"})," is parsed as GNU Make syntax\nwhen loaded via ",(0,r.jsx)(n.code,{children:"make backend"})," / ",(0,r.jsx)(n.code,{children:"make dev"})," \u2014 a literal ",(0,r.jsx)(n.code,{children:"$"})," there is\ninterpreted as a Make variable reference and silently corrupted."]}),"\n",(0,r.jsxs)(n.p,{children:["The matcher applies to the parameter ",(0,r.jsx)(n.em,{children:"name"}),", not its value, and only one matcher is in effect at a time.\nTo fall back to the tracer's own weaker default, set this to an empty value; the tracer logs a ",(0,r.jsx)(n.code,{children:"Couldn't parse INSTANA_SECRETS"})," warning each time you do."]}),"\n",(0,r.jsx)(n.admonition,{type:"warning",children:(0,r.jsxs)(n.p,{children:["Don't set ",(0,r.jsx)(n.code,{children:"INSTANA_EXTRA_HTTP_HEADERS"}),".\nIt tells the tracer to copy the named request headers onto spans, and OpenRAG's headers carry session cookies, API keys, and bearer tokens."]})}),"\n",(0,r.jsxs)(n.p,{children:["If your deployment can't export this data at all, keep ",(0,r.jsx)(n.code,{children:"INSTANA_ENABLED=false"}),". There is no partial mode in which the backend is traced but request metadata stays local."]}),"\n",(0,r.jsx)(n.h2,{id:"before-you-begin",children:"Before you begin"}),"\n",(0,r.jsx)(n.p,{children:"You need an Instana tenant, an agent key, and the ingress endpoint for your tenant's region."}),"\n",(0,r.jsxs)(n.p,{children:["To get the key and endpoint, open your Instana tenant and go to ",(0,r.jsx)(n.strong,{children:"Infrastru
1cture > Add a Host"}),", then select the Docker installation method.\nThe command that the wizard shows you contains both values as ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_KEY"})," and ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_ENDPOINT"}),"."]}),"\n",(0,r.jsxs)(n.p,{children:["The ingress endpoint is region-specific, such as ",(0,r.jsx)(n.code,{children:"ingress-coral-saas.instana.io"}),".\nUse the value from your own wizard rather than the OpenRAG default, or the agent connects to the wrong region and reports no data."]}),"\n",(0,r.jsx)(n.h3,{id:"install-the-tracer",children:"Install the tracer"}),"\n",(0,r.jsxs)(n.p,{children:["The Instana Python tracer is an optional dependency, declared as the ",(0,r.jsx)(n.code,{children:"apm"})," extra, so that deployments that never enable APM don't install it."]}),"\n",(0,r.jsxs)(n.p,{children:["The OpenRAG Backend container image includes the extra already, so ",(0,r.jsx)(n.a,{href:"/docker",children:"self-managed"})," and Kubernetes deployments need nothing extra."]}),"\n",(0,r.jsxs)(n.p,{children:["If you run the backend directly from source, such as ",(0,r.jsx)(n.code,{children:"make backend"})," for ",(0,r.jsx)(n.a,{href:"/support/contribute",children:"contributors"}),", install the extra once:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"uv sync --extra apm\n"})}),"\n",(0,r.jsxs)(n.p,{children:["If ",(0,r.jsx)(n.code,{children:"INSTANA_ENABLED"})," is set and the package isn't installed, the backend logs a warning at startup and runs without tracing rather than failing to start."]}),"\n",(0,r.jsx)(n.h2,{id:"enable-tracing",children:"Enable tracing in a self-managed deployment"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Add the Instana variables to your ",(0,r.jsx)(n.code,{children:".env"})," file:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:'INSTANA_ENABLED=true\nINSTANA_AGENT_KEY=YOUR_AGENT_KEY\nINSTANA_AGENT_ENDPOINT=ingress-coral-saas.instana.io\nINSTANA_AGENT_ENDPOINT_PORT=443\nINSTANA_SERVICE_NAME="OpenRAG Backend"\nINSTANA_ZONE="openrag-dev"\n'})}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_SERVICE_NAME"})," and ",(0,r.jsx)(n.code,{children:"INSTANA_ZONE"})," are optional, but they make OpenRAG easier to find in the Instana UI.\nFor the complete list of variables, see ",(0,r.jsx)(n.a,{href:"/reference/configuration#observability-and-apm-settings",children:"Observability and APM settings"}),"."]}),"\n",(0,r.jsx)(n.admonition,{type:"warning",children:(0,r.jsxs)(n.p,{children:["Don't set the optional variables to an empty value.\nThe tracer checks ",(0,r.jsx)(n.code,{children:"INSTANA_SERVICE_NAME"}),", ",(0,r.jsx)(n.code,{children:"INSTANA_ZONE"}),", and ",(0,r.jsx)(n.code,{children:"INSTANA_LOG_LEVEL"})," for presence rather than for a value, so ",(0,r.jsx)(n.code,{children:"INSTANA_SERVICE_NAME="})," registers a service with a blank name, and ",(0,r.jsx)(n.code,{children:"INSTANA_LOG_LEVEL="})," logs an ",(0,r.jsx)(n.code,{children:"Unknown INSTANA_LOG_LEVEL"})," warning on every start.\nTo leave a variable at its default, comment it out or remove it."]})}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Start the Instana host agent:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"make instana-agent-up\n"})}),"\n",(0,r.jsxs)(n.p,{children:["The target fails if ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_KEY"})," isn't set. For what this agent does to your machine, see ",(0,r.jsx)(n.a,{href:"#local-host-agent",children:"Run the local host agent"}),"."]}),"\n",(0,r.jsxs)(n.p,{children:["If your agent runs somewhere else already, skip this step and set ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_HOST"})," to that agent's address instead."]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.a,{href:"/manage-services",children:"Restart the OpenRAG services"})," so the backend picks up the new configuration."]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_ENABLED"})," is read once, at process start. Changing it requires a restart of the backend, not just a page reload."]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Confirm that the backend loaded the tracer:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"docker
1compose logs openrag-backend | grep -i instana\n"})}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["In your Instana tenant, open ",(0,r.jsx)(n.strong,{children:"Applications"})," and look for your ",(0,r.jsx)(n.code,{children:"INSTANA_SERVICE_NAME"})," value, or open ",(0,r.jsx)(n.strong,{children:"Analyze > Traces"})," and send a request through OpenRAG."]}),"\n",(0,r.jsx)(n.p,{children:"Traces can take a minute or two to appear after the agent first connects."}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["To turn instrumentation off again, set ",(0,r.jsx)(n.code,{children:"INSTANA_ENABLED=false"})," or remove it, restart the services, and run ",(0,r.jsx)(n.code,{children:"make instana-agent-down"}),"."]}),"\n",(0,r.jsx)(n.h2,{id:"local-host-agent",children:"Run the local host agent"}),"\n",(0,r.jsxs)(n.p,{children:["OpenRAG ships the host agent as an opt-in Docker Compose service under the ",(0,r.jsx)(n.code,{children:"instana"})," profile, so it never starts as part of ",(0,r.jsx)(n.code,{children:"make dev"})," or ",(0,r.jsx)(n.code,{children:"docker compose up"}),"."]}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Command"}),(0,r.jsx)(n.th,{children:"Description"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"make instana-agent-up"})}),(0,r.jsxs)(n.td,{children:["Starts the ",(0,r.jsx)(n.code,{children:"instana-agent"})," container. Requires ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_KEY"}),". The agent listens on ",(0,r.jsx)(n.code,{children:"127.0.0.1:42699"})," and forwards to ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_ENDPOINT"}),"."]})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"make instana-agent-down"})}),(0,r.jsxs)(n.td,{children:["Stops the ",(0,r.jsx)(n.code,{children:"instana-agent"})," container."]})]})]})]}),"\n",(0,r.jsxs)(n.admonition,{type:"warning",children:[(0,r.jsxs)(n.p,{children:["The host agent container runs as ",(0,r.jsx)(n.code,{children:"privileged"}),", in the host PID namespace, with host networking and a read-write mount of the Docker socket.\nThe agent needs this access to monitor host and container infrastructure, and the same access is effectively root on the machine that runs it."]}),(0,r.jsx)(n.p,{children:"Use these targets on a local development machine only.\nDon't run them on a shared or production host. Deploy the agent there through whatever process your organization uses for host agents."})]}),"\n",(0,r.jsx)(n.admonition,{type:"info",children:(0,r.jsxs)(n.p,{children:["The agent service uses ",(0,r.jsx)(n.code,{children:"network_mode: host"}),", which is supported on Linux Docker hosts, and on Docker Desktop only when host networking is enabled.\nOn other platforms, the container can start without publishing port ",(0,r.jsx)(n.code,{children:"42699"})," on the host, and the backend then reports no traces even though the agent appears to be running.\nOn those platforms, install the agent natively with the wizard's non-Docker instructions, or point ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_HOST"})," at an agent running elsewhere."]})}),"\n",(0,r.jsxs)(n.p,{children:["The agent image is pinned in ",(0,r.jsx)(n.code,{children:"docker-compose.yml"})," rather than tracked on ",(0,r.jsx)(n.code,{children:"latest"}),", because a privileged container shouldn't change underneath you on a re-pull.\nTo try a newer agent without editing Compose, set ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_IMAGE_TAG"})," to a tag from ",(0,r.jsx)(n.a,{href:"https://icr.io/v2/instana/agent/tags/list",children:"the agent's tag list"}),"."]}),"\n",(0,r.jsx)(n.h2,{id:"kubernetes",children:"Enable tracing in Kubernetes"}),"\n",(0,r.jsxs)(n.p,{children:["The OpenRAG Helm chart configures the backend tracer under ",(0,r.jsx)(n.code,{children:"backend.instana"}),":"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'backend:\n instana:\n enabled: true\n agentPort: 42699\n serviceName: "OpenRAG Backend"\n zone: "openrag-cpd"\n'})}),"\n",(0,r.jsxs)(n.p,{children:["Leave ",(0,r.jsx)(n.code,{children:"agentHost"})," empty. The chart then injects the pod's own node IP through the Kubernetes Downward API, which is the address that the standard per-node agent DaemonSet expects.\nSet ",(0,r.jsx)(n.code,{children:"agentHost"})," explicitly only if your agent isn't a DaemonSet and is reached at a fixed address, such as through a Service."]}),"\n",(0,r.jsxs)(n.p,{children:["The same presence rule applies as in ",(0,r.jsx)(n.code,{children:".env"}),": ",(0,r.jsx)(n.code,{children:"serviceName"}),", ",(0,r.jsx)(n.code,{children:"zone"}),", and ",(0,r.jsx)(n.code,{children:"logLevel"})," are omitted from the rendered configuration when they're empty, so leave them empty rather than setting them to ",(0,r.jsx)(n.code,{children:'""'})," deliberately."]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"tracingDisable"}),", ",(0,r.jsx)(n.code,{children:"stackTrace"}),", and ",(0,r.jsx)(n.code,{children:"secrets"})," don't follow that rule. All three ship with a real, non-empty chart default (",(0,r.jsx)(n.code,{children:"logging"}),", ",(0,r.jsx)(n.code,{children:"error"}),", and the redaction regex in ",(0,r.jsx)(n.a,{href:"#redact-query-parameters",children:"Redact query parameters"})," \u2014 see also ",(0,r.jsx)(n.a,{href:"#performance-guardrails",children:"Performance guardrails"}),"), so leaving them out of your values override doesn't disable the guardrail or weaken the redaction;
1 the chart's default still gets written. To get the tracer's own default instead, override them explicitly: ",(0,r.jsx)(n.code,{children:'tracingDisable: ""'}),", ",(0,r.jsx)(n.code,{children:'stackTrace: "all"'}),", or ",(0,r.jsx)(n.code,{children:'secrets: ""'}),"."]}),"\n",(0,r.jsx)(n.admonition,{type:"important",children:(0,r.jsxs)(n.p,{children:["The OpenRAG chart doesn't install an Instana agent.\nInstall one separately with IBM's ",(0,r.jsx)(n.code,{children:"instana-agent"})," Helm chart or operator.\nThe agent is a privileged, host-PID DaemonSet, so installing it is a cluster administration task."]})}),"\n",(0,r.jsx)(n.h3,{id:"operator",children:"Enable tracing with the OpenRAG operator"}),"\n",(0,r.jsxs)(n.p,{children:["For deployments managed by the ",(0,r.jsx)(n.a,{href:"https://github.com/langflow-ai/openrag/tree/main/kubernetes/operator",children:"OpenRAG operator"}),", set the variables in the ",(0,r.jsx)(n.code,{children:"OpenRAG"})," custom resource:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'apiVersion: openr.ag/v1alpha1\nkind: OpenRAG\nmetadata:\n name: my-openrag\nspec:\n backend:\n env:\n - name: INSTANA_ENABLED\n value: "true"\n - name: INSTANA_SERVICE_NAME\n value: "OpenRAG Backend"\n - name: INSTANA_ZONE\n value: "openrag-cpd"\n'})}),"\n",(0,r.jsxs)(n.p,{children:["Don't set ",(0,r.jsx)(n.code,{children:"INSTANA_AGENT_HOST"}),". When ",(0,r.jsx)(n.code,{children:"INSTANA_ENABLED"})," is truthy, the operator injects it from the pod's node IP, and setting it in the custom resource suppresses that injection."]}),"\n",(0,r.jsx)(n.h2,{id:"tracer-logging",children:"Configure the tracer's own logging"}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_LOG_LEVEL"})," controls how much the tracer itself logs, as one of ",(0,r.jsx)(n.code,{children:"debug"}),", ",(0,r.jsx)(n.code,{children:"info"}),", ",(0,r.jsx)(n.code,{children:"warn"}),", or ",(0,r.jsx)(n.code,{children:"error"}),".\nIt's unrelated to ",(0,r.jsx)(n.a,{href:"/reference/configuration#system-settings",children:(0,r.jsx)(n.code,{children:"LOG_LEVEL"})}),", which controls OpenRAG's application logs.\nSet it to ",(0,r.jsx)(n.code,{children:"debug"})," when you're diagnosing why traces don't reach Instana, and leave it unset otherwise."]}),"\n",(0,r.jsx)(n.h2,{id:"performance-guardrails",children:"Performance guardrails"}),"\n",(0,r.jsxs)(n.p,{children:["Two of the tracer's own defaults are expensive in a long-running backend, so OpenRAG ships a different value for each. Both are set for you in ",(0,r.jsx)(n.code,{children:".env.example"}),", ",(0,r.jsx)(n.code,{children:"docker-compose.yml"}),", the Helm chart, and the operator \u2014 you only need to change them to opt out."]}),"\n",(0,r.jsx)(n.h3,{id:"logging-spans",children:"Logging spans"}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_TRACING_DISABLE=logging"})," stops the tracer from creating a span for every ",(0,r.jsx)(n.code,{children:"WARNING"})," and ",(0,r.jsx)(n.code,{children:"ERROR"})," that OpenRAG logs while handling a request."]}),"\n",(0,r.jsx)(n.p,{children:"Left enabled, each of those spans records an event in a list that the tracer shares across all spans and never frees, and every later logging span walks the whole list before it's reported. The cost per logged warning therefore climbs for as long as the process runs, and the memory is never returned. Measured on the OpenRAG backend, a single warning logged inside a request trace costs about 1 ms with logging spans on and about 26 \u03bcs with them off."}),"\n",(0,r.jsxs)(n.p,{children:["You lose no information by disabling them. OpenRAG's own structured logs, controlled by ",(0,r.jsx)(n.a,{href:"/reference/configuration#system-settings",children:(0,r.jsx)(n.code,{children:"LOG_LEVEL"})}),", already record the same warnings and errors, with more context than the span carries."]}),"\n",(0,r.jsx)(n.p,{children:"To trace logging spans anyway, set the variable to an empty value rather than removing it:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"INSTANA_TRACING_DISABLE=\n"})}),"\n",(0,r.jsx)(n.h3,{id:"exit-span-stack-traces",children:"Stack traces on exit spans"}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"INSTANA_STACK_TRACE=error"})," limits stack capture to spans that actually failed."]}),"\n",(0,r.jsxs)(n.p,{children:["An exit span is created for every call OpenRAG makes out of the process \u2014 every OpenSearch query, every request to Langflow, and every database query. At the tracer's own default of ",(0,r.jsx)(n.code,{children:"all"}),", each one captures a full Python stack trace and reads the matching source lines from disk. OpenRAG's async request stack
1is deep enough that this costs 200 to 350 \u03bcs per exit span, and a single chat request makes tens of them."]}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"error"})," keeps the stack trace where it's diagnostic and drops it from the successful path. To capture stacks on every exit span, set ",(0,r.jsx)(n.code,{children:"all"})," \u2014 not an empty value, which the tracer rejects with a warning and then falls back to ",(0,r.jsx)(n.code,{children:"all"})," anyway:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"INSTANA_STACK_TRACE=all\n"})}),"\n",(0,r.jsx)(n.h2,{id:"variables",children:"Environment variable reference"}),"\n",(0,r.jsxs)(n.p,{children:["For every variable in this feature, including defaults, see ",(0,r.jsx)(n.a,{href:"/reference/configuration#observability-and-apm-settings",children:"Observability and APM settings"}),"."]})]})}function l(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(h,{...e})}):h(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.