https://www.lingo-db.com/assets/js/7a754141.df00e9be.js

1"use strict";(globalThis.webpackChunklingodb_docs||=[]).push([[8395],{229(e,n,i){i.r(n),i.d(n,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>r,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"ForDevelopers/Debugging","title":"Debugging & Profiling","description":"Compared to interpreted execution engines, compiling engines come with many advantages but also some challenges.","source":"@site/versioned_docs/version-0.0.3/ForDevelopers/Debugging.md","sourceDirName":"ForDevelopers","slug":"/ForDevelopers/Debugging","permalink":"/docs/0.0.3/ForDevelopers/Debugging","draft":false,"unlisted":false,"editUrl":"https://github.com/lingo-db/lingo-db.github.io/tree/main/versioned_docs/version-0.0.3/ForDevelopers/Debugging.md","tags":[],"version":"0.0.3","frontMatter":{"title":"Debugging & Profiling"},"sidebar":"docsSidebar","previous":{"title":"Contributing","permalink":"/docs/0.0.3/ForDevelopers/Contributing"},"next":{"title":"Dependencies","permalink":"/docs/0.0.3/ForDevelopers/Dependencies"}}');var o=i(4848),s=i(8453);const r={title:"Debugging & Profiling"},a=void 0,l={},d=[{value:"Guide: Profiling queries",id:"guide-profiling-queries",level:2},{value:"Guide: Debugging",id:"guide-debugging",level:2},{value:"Components for Debugging and Profiling",id:"components-for-debugging-and-profiling",level:2},{value:"Location Tracking in MLIR",id:"location-tracking-in-mlir",level:3},{value:"Snapshotting",id:"snapshotting",level:3},{value:"Special Compiler Backends",id:"special-compiler-backends",level:3},{value:"LLVM-Debug Backend",id:"llvm-debug-backend",level:4},{value:"C++-Backend",id:"c-backend",level:4},{value:"Lightweight Tracing",id:"lightweight-tracing",level:3}];function c(e){const n={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsxs)(n.p,{children:["Compared to interpreted execution engines, compiling engines come with many advantages but also some challenges.\nEspecially debugging and profiling can become a challenge, as one not only needs to debug and profile the engine code, but also the generated code.\nPossible solutions to these problems have been discussed before for debugging ",(0,o.jsx)(n.a,{href:"https://ieeexplore.ieee.org/document/8667737",children:"Hyper"})," and ",(0,o.jsx)(n.a,{href:"https://dl.acm.org/doi/abs/10.1145/3395032.3395321",children:"Umbra"})," and ",(0,o.jsx)(n.a,{href:"https://dl.acm.org/doi/abs/10.1145/3447786.3456254",children:"profiling Umbra"}),"."]}),"\n",(0,o.jsx)(n.h2,{id:"guide-profiling-queries",children:"Guide: Profiling queries"}),"\n",(0,o.jsxs)(n.p,{children:["For profiling queries LingoDB comes with a ",(0,o.jsx)(n.em,{children:"ct"})," tool that collects several metrics.\nFor the following instructions, we assume that LingoDB was built in Release mode with debugging informations (",(0,o.jsx)(n.code,{children:"build/lingodb-relwithdebinfo/.buildstamp"})," )."]}),"\n",(0,o.jsxs)(n.ol,{children:["\n",(0,o.jsxs)(n.li,{children:["Run the ct.py script with query and dataset: ",(0,o.jsx)(n.code,{children:"python3 tools/ct/ct.py resources/sql/tpch/1.sql resources/data/tpch-1/"}),". If the build directory is not ",(0,o.jsx)(n.code,{children:"build/lingodb-relwithdebinfo"}),", it can be supplied with the ",(0,o.jsx)(n.code,{children:"BIN_DIR"})," environment variable"]}),"\n",(0,o.jsxs)(n.li,{children:["Open the resulting ",(0,o.jsx)(n.code,{children:"ct.json"})," file with the ",(0,o.jsx)(n.a,{href:"https://ct.lingo-db.com",children:"CT viewer"})," and explore it in detail"]}),"\n"]}),"\n",(0,o.jsx)(n.h2,{id:"guide-debugging",children:"Guide: Debugging"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsxs)(n.li,{children:["If the compilation fails: Use ",(0,o.jsx)(n.a,{href:"#snapshotting",children:"Snapshotting"})," to identify the broken/problematic pass. Then run the pass isolated 

1with ",(0,o.jsx)(n.a,{href:"/docs/0.0.3/GettingStarted/CommandLineTools#performing-optimizations-and-lowerings",children:"mlir-db-opt"})," for detailed debugging (e.g., with gdb)."]}),"\n",(0,o.jsxs)(n.li,{children:["If compilation succeeds but execution fails in/because generated code: First check if the error persists when switching to the ",(0,o.jsx)(n.a,{href:"#c-backend",children:"C++-Backend"})," if possible (i.e., all MLIR operations are supported)","\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:"If yes: debug with this backend."}),"\n",(0,o.jsxs)(n.li,{children:["If not: you should use the ",(0,o.jsx)(n.a,{href:"#llvm-debug-backend",children:"LLVM Debug Backend"})]}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,o.jsx)(n.h2,{id:"components-for-debugging-and-profiling",children:"Components for Debugging and Profiling"}),"\n",(0,o.jsx)(n.h3,{id:"location-tracking-in-mlir",children:"Location Tracking in MLIR"}),"\n",(0,o.jsxs)(n.p,{children:["In MLIR, every operation is associated with a ",(0,o.jsx)(n.em,{children:"Location"}),", that must be provided during operation creation.\nWhile it is possible to provide a ",(0,o.jsx)(n.em,{children:"Unknown Location"}),", it should be avoided.\nWhen parsing a MLIR file, MLIR automatically annotates the parsed operations with the corresponding file locations.\nWhen new operations are created during a pass they are usually annotated with the location of the current operation that is transformed or lowered.\n",(0,o.jsx)(n.strong,{children:"All passes in LingoDB ensure that correct locations are set afterwards."})]}),"\n",(0,o.jsx)(n.h3,{id:"snapshotting",children:"Snapshotting"}),"\n",(0,o.jsxs)(n.p,{children:["MLIR already comes with a ",(0,o.jsx)(n.code,{children:"LocationSnapshotPass"})," that takes an operation (e.g. a MLIR Module) and writes it to disk, including the annotated locations.\nThen, this file is now read back in, now annotating the locations ",(0,o.jsx)(n.em,{children:"according to the location inside this newly written file"}),"."]}),"\n",(0,o.jsxs)(n.p,{children:["If enabled (cf ",(0,o.jsx)(n.a,{href:"/docs/0.0.3/ForDevelopers/Settings",children:"Settings"})," ), LingoDB performs multiple location snapshots on after every or selected (important) MLIR passes."]}),"\n",(0,o.jsx)(n.p,{children:"Using this snapshot files, we can track the origin of any operation, by recursively following the following steps"}),"\n",(0,o.jsxs)(n.ol,{children:["\n",(0,o.jsx)(n.li,{children:"get the origin location of the current operation by looking in the appropriate snapshot file"}),"\n",(0,o.jsx)(n.li,{children:"find the origin operation by going to this location"}),"\n"]}),"\n",(0,o.jsx)(n.h3,{id:"special-compiler-backends",children:"Special Compiler Backends"}),"\n",(0,o.jsx)(n.p,{children:"In addition to location tracking and snapshotting, LingoDB implements two special compiler backends for debugging."}),"\n",(0,o.jsx)(n.h4,{id:"llvm-debug-backend",children:"LLVM-Debug Backend"}),"\n",(0,o.jsxs)(n.p,{children:["Instead of using the standard LLVM backend, another LLVM-based backend can be used that adds debug information and performs no optimizations.\nThis backend is selected by setting the environment variable ",(0,o.jsx)(n.code,{children:"LINGODB_EXECUTION_MODE=DEBUGGING"}),".\nDuring the execution, standard debuggers like ",(0,o.jsx)(n.code,{children:"gdb"})," will then point to the corresponding operation in the last snapshot that was performed\nThis enables basic tracking of problematic operations, but advanced debugging will remain difficult."]}),"\n",(0,o.jsx)(n.h4,{id:"c-backend",children:"C++-Backend"}),"\n",(0,o.jsxs)(n.p,{children:["For more advanced debugging, a ",(0,o.jsx)(n.em,{children:"C++-Backend"})," can be used by setting ",(0,o.jsx)(n.code,{children:"LINGODB_EXECUTION_MODE=C"}),".\nThis backend directly translates a fixed set of low-level generic MLIR operations to C++ statements and functions that are written to a file called ",(0,o.jsx)(n.code,{children:"mlir-c-module.cpp"}),".\nNext, LingoDB automatically invokes ",(0,o.jsx)(n.code,{children:"clang++"})," (must be installed!) with ",(0,o.jsx)(n.code,{children:"-O0"})," and ",(0,o.jsx)(n.code,{children:"-g"})," to compile this C++ file into a shared library with debug informations.\nThis shared library is then loaded with ",(0,o.jsx)(n.code,{children:"dlopen"})," and the main function is called.\nThus, the generated code can be debugged as any usual C++ program.\nTo help with tracking an error to higher-level MLIR operations, each C++ statement is preceeded with a comment containing the original operation and it's location."]}),"\n",(0,o.jsx)(n.h3,{id:"lightweight-tracing",children:"Lightweight Tracing"}),"\n",(0,o.jsxs)(n.p,{children:["When compiled as ",(0,o.jsx)(n.code,{children:"RelWithDebInfo"}),", LingoDB will produce a trace file with events (type, start timestamp, duration, thread) as trace.json.\nThis trace file can then be opened with the ",(0,o.jsx)(n.a,{href:"https://ct.lingo-db.com",children:"CT Viewer"})]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(c,{...e})}):c(e)}},8453(e,n,i){i.d(n,{R:()=>r,x:()=>a});var t=i(6540);const o={},s=t.createContext(o);function r(e){const n=t.useContext(s);return t.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:r(e.components),t.createElement(s.Provider,{value:n},e.children)}}}]);