1"use strict";(self.webpackChunksuccinct_docs=self.webpackChunksuccinct_docs||[]).push([[5437],{12368:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>o,default:()=>u,frontMatter:()=>a,metadata:()=>i,toc:()=>d});const i=JSON.parse('{"id":"provers/cluster-development/distributed-tracing","title":"Distributed Tracing","description":"SP1 Cluster utilizes OpenTelemetry distributed tracing to instrument the entire proving system. Tracing can be helpful to identify latency/throughput bottlenecks across the distributed cluster and find issues (e.g. artifact data transfer, trace generation, coordinator API latency) in the lifetime of a particular proof.","source":"@site/docs/provers/cluster-development/distributed-tracing.md","sourceDirName":"provers/cluster-development","slug":"/provers/cluster-development/distributed-tracing","permalink":"/docs/provers/cluster-development/distributed-tracing","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":1,"frontMatter":{"title":"Distributed Tracing","sidebar_position":1,"toc_min_heading_level":2,"toc_max_heading_level":3},"sidebar":"proversSidebar","previous":{"title":"Bidder","permalink":"/docs/provers/cluster-development/bidder"},"next":{"title":"Prometheus Metrics","permalink":"/docs/provers/cluster-development/prometheus-metrics"}}');var s=n(74848),r=n(28453);const a={title:"Distributed Tracing",sidebar_position:1,toc_min_heading_level:2,toc_max_heading_level:3},o=void 0,c={},d=[{value:"Setup",id:"setup",level:2},{value:"Navigating Traces",id:"navigating-traces",level:2},{value:"Inspecting Task Details",id:"inspecting-task-details",level:2},{value:"Instrumentation Code",id:"instrumentation-code",level:2}];function l(e){const t={a:"a",admonition:"admonition",code:"code",h2:"h2",li:"li",p:"p",ul:"ul",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsxs)(t.p,{children:["SP1 Cluster utilizes ",(0,s.jsx)(t.a,{href:"https://opentelemetry.io/
1docs/concepts/signals/traces/",children:"OpenTelemetry distributed tracing"})," to instrument the entire proving system. Tracing can be helpful to identify latency/throughput bottlenecks across the distributed cluster and find issues (e.g. artifact data transfer, trace generation, coordinator API latency) in the lifetime of a particular proof."]}),"\n",(0,s.jsx)(t.h2,{id:"setup",children:"Setup"}),"\n",(0,s.jsx)(t.p,{children:"To enable tracing, you will need to set the following environment variables on CPU and GPU workers:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:(0,s.jsx)(t.code,{children:"TRACING_ENABLED=true"})}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"OTLP_ENDPOINT"})," to the endpoint of your tracing collector/exporter (e.g. ",(0,s.jsx)(t.a,{href:"https://grafana.com/docs/grafana-cloud/send-data/traces/set-up/traces-with-alloy/#configure-grafana-alloy-to-send-traces-to-grafana-cloud",children:"Grafana Alloy"}),")."]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"RUST_LOG=debug"})," for maximum trace visibility."]}),"\n"]}),"\n",(0,s.jsx)(t.h2,{id:"navigating-traces",children:"Navigating Traces"}),"\n",(0,s.jsxs)(t.p,{children:["Once you can successfully gather traces and view them in your tracing backend, you should see a root trace with name ",(0,s.jsx)(t.code,{children:"Prove <proof ID>"})," for each proof request. In this root trace, you should see a span for every single task executed during the duration of the proof. Each task span depicts how long that task took to complete once it began running on a worker node."]}),"\n",(0,s.jsx)(t.admonition,{type:"info",children:(0,s.jsx)(t.p,{children:'We recommend using "Collapse all" and then "Expand +1" to see the spans better.'})}),"\n",(0,s.jsx)(t.h2,{id:"inspecting-task-details",children:"Inspecting Task Details"}),"\n",(0,s.jsx)(t.p,{children:'Each task span should be linked to another root span for the task, so you should be able to click a link icon on the span and then "View linked span". The task root span is also linked in the reverse direction.'}),"\n",(0,s.jsxs)(t.p,{children:["Each task span should also contain span attribute ",(0,s.jsx)(t.code,{children:"task_id"})," and resource attribute ",(0,s.jsx)(t.code,{children:"node.id"})," to aid in debugging."]}),"\n",(0,s.jsx)(t.h2,{id:"instrumentation-code",children:"Instrumentation Code"}),"\n",(0,s.jsxs)(t.p,{children:["To add more tracing detail in the worker code, you can simply add ",(0,s.jsx)(t.code,{children:"tracing::info!"})," spans to the code. To hide detail, you can change spans to be ",(0,s.jsx)(t.code,{children:"trace!"})," level."]}),"\n",(0,s.jsx)(t.p,{children:"Note that span context does not automatically propagate into async (Tokio) tasks. You can create a span outside of the task and move it into the task to ensure child spans in the task are under the parent span."}),"\n",(0,s.jsx)(t.p,{children:"Note that adding too many spans can cause performance issues and make traces hard to read."})]})}function u(e={}){const{wrapper:t}={...(0,r.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(l,{...e})}):l(e)}},28453:(e,t,n)=>{n.d(t,{R:()=>a,x:()=>o});var i=n(96540);const s={},r=i.createContext(s);function a(e){const t=i.useContext(r);return i.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function o(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),i.createElement(r.Provider,{value:t},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.