https://bpftrace.org/assets/js/660d996e.6a81af08.js

1"use strict";(self.webpackChunktest_website=self.webpackChunktest_website||[]).push([[2290],{4246:(e,s,n)=>{n.r(s),n.d(s,{assets:()=>a,contentTitle:()=>c,default:()=>d,frontMatter:()=>l,metadata:()=>t,toc:()=>o});const t=JSON.parse('{"type":"mdx","permalink":"/hol/system-calls","source":"@site/src/pages/hol/system-calls.md","title":"2. Working with system calls","description":"In this lab you will experiment with tracing system call interfaces. As this is the primary mechanism with which a process interacts with the kernel, it can often be of interest when investigating systemic and/or application behavior.","frontMatter":{},"unlisted":false}');var r=n(4848),i=n(8453);const l={},c="2. Working with system calls",a={},o=[{value:"Syscall probe naming format",id:"syscall-probe-naming-format",level:2},{value:"Exercise 2.1",id:"exercise-21",level:3},{value:"Syscall entry probes",id:"syscall-entry-probes",level:2},{value:"Syscall return probes",id:"syscall-return-probes",level:2},{value:"Exercises",id:"exercises",level:2},{value:"Exercises 2.2: <code>mmap(2)</code>",id:"exercises-22-mmap2",level:3},{value:"Exercises 2.3: <code>open(2)/openat(2)</code>",id:"exercises-23-open2openat2",level:3},{value:"Exercises 2.4: <code>close(2)</code>",id:"exercises-24-close2",level:3},{value:"Further Reading",id:"further-reading",level:2},{value:"Back to HOL Intro",id:"back-to-hol-intro",level:2}];function h(e){const s={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",hr:"hr",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,i.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(s.header,{children:(0,r.jsx)(s.h1,{id:"2-working-with-system-calls",children:"2. Working with system calls"})}),"\n",(0,r.jsx)(s.p,{children:"In this lab you will experiment with tracing system call interfaces. As this is the primary mechanism with which a process interacts with the kernel, it can often be of interest when investigating systemic and/or application behavior."}),"\n",(0,r.jsxs)(s.p,{children:["The number of system calls may vary from kernel to kernel and we can see which exist on our system with ",(0,r.jsx)(s.code,{children:"bpftrace -l"}),":"]}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"sudo bpftrace -l 'tracepoint:syscalls:sys_enter*'\n"})}),"\n",(0,r.jsx)(s.p,{children:"Example output:"}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"tracepoint:syscalls:sys_enter_accept\ntracepoint:syscalls:sys_enter_accept4\ntracepoint:syscalls:sys_enter_access\ntracepoint:syscalls:sys_enter_add_key\ntracepoint:syscalls:sys_enter_adjtimex\n...\n"})}),"\n",(0,r.jsx)(s.p,{children:"You should have in excess of 300 system calls to choose from!"}),"\n",(0,r.jsx)(s.h2,{id:"syscall-probe-naming-format",children:"Syscall probe naming format"}),"\n",(0,r.jsxs)(s.p,{children:["Each syscall has two distinct probes: an ",(0,r.jsx)(s.strong,{children:"enter"})," probe which is fired when a system call is called and an ",(0,r.jsx)(s.strong,{children:"exit"})," probe which is fired on return from a system call. The format of the two different types of syscall probe are:"]}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"tracepoint:syscalls:sys_enter_<syscall_name>\ntracepoint:syscalls:sys_exit_<syscall_name>\n"})}),"\n",(0,r.jsxs)(s.p,{children:["For example, the ",(0,r.jsx)(s.code,{children:"open(2)"})," system call probes are:"]}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"tracepoint:syscalls:sys_enter_open\ntracepoint:syscalls:sys_exit_open\n"})}),"\n",(0,r.jsx)(s.h3,{id:"exercise-21",children:"Exercise 2.1"}),"\n",(0,r.jsxs)(s.p,{children:["Does ",(0,r.jsx)(s.code,{children:"t:syscalls:sys_exit_exit"})," exist? If so, when will it fire?"]}),"\n",(0,r.jsx)(s.h2,{id:"syscall-entry-probes",children:"Syscall entry probes"}),"\n",(0,r.jsxs)(s.p,{children:["The arguments for a system call probe are made available through ",(0,r.jsxs)(s.a,{href:"/docs/release_023/stdlib#builtins",children:["the \ud83d\udcd6 ",(0,r.jsx)(s.code,{children:"args"})," builtin structure"]}),". For example, according to the man page for ",(0,r.jsx)(s.code,{children:"write(2)"}),", the syscall has 3 arguments: ",(0,r.jsx)(s.code,{children:"int fd"}),", ",(0,r.jsx)(s.code,{children:"const char * buf"})," and ",(0,r.jsx)(s.code,{children:"size_t count"}),". We can verify that with the ",(0,r.jsx)(s.code,{children:"-lv"})," options to ",(0,r.jsx)(s.code,{children:"bpftrace"}),":"]}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"sudo bpftrace -lv 't:syscalls:sys_enter_write'\n"})}),"\n",(0,r.jsx)(s.p,{children:"Example output:"}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:'tracepoint:syscalls:sys_enter_write\n    int __syscall_nr\n    unsigned int fd\n    const char __attribute__((btf_type_tag("user"))) * buf\n    size_t count\n'})}),"\n",(0,r.jsx)(s.p,{children:"Things to note:"}),"\n",(0,r.jsxs)(s.ul,{children:["\n",(0,r.jsxs)(s.li,{children:["We've used the abbreviated name for ",(0,r.jsx)(s.code,{children:"tracepoint"}),' above - simply "',(0,r.jsx)(s.code,{children:"t"}),'". Every probe type has an abbreviated format.']}

1),"\n",(0,r.jsxs)(s.li,{children:["You can ignore the ",(0,r.jsx)(s.code,{children:"__attribute__"})," tag on the ",(0,r.jsx)(s.code,{children:"buf"})," parameter. This is kernel implementation detail you don't need to worry about. You may not see the attribute tag on your kernel."]}),"\n",(0,r.jsxs)(s.li,{children:["Those with a keen eye may have noted that we have an extra parameter - ",(0,r.jsx)(s.code,{children:"int __syscall_nr"}),". Again, this is just an implementation detail that has been exposed to you and you'll probably have little use for it. It's the system call number assigned to this system call in the kernel (more detail in \"Further Reading\" - see below)"]}),"\n"]}),"\n",(0,r.jsxs)(s.p,{children:["To access an argument, we reference it through ",(0,r.jsxs)(s.a,{href:"/docs/release_023/stdlib#builtins",children:["the \ud83d\udcd6 ",(0,r.jsx)(s.code,{children:"args"})," builtin"]})," using its name, e.g, ",(0,r.jsx)(s.code,{children:"args.buf"}),". In the following example we capture the first 32KB bytes (or less) of any buffer being sent to file descriptor 2 which is usually ",(0,r.jsx)(s.code,{children:"stderr"})," (although it's obviously not guaranteed to be that)."]}),"\n",(0,r.jsxs)(s.p,{children:["Create a new file named ",(0,r.jsx)(s.code,{children:"write.bt"})," and paste this code:"]}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{children:'config = {\n  max_strlen = 32768; /* 32KB - max string size */\n}\n\n/* fd 2 might not be a processes stderr but it\'s good fun nonetheless */\ntracepoint:syscalls:sys_enter_write\n/args.fd == 2/\n{\n        printf("%s: %s\\n", comm,  str(args.buf));\n}\n'})}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"sudo bpftrace ./write.bt\n"})}),"\n",(0,r.jsx)(s.p,{children:"Example output:"}),"\n",(0,r.jsx)(s.pre,{children:(0,r.jsx)(s.code,{className:"language-sh",children:"Collection-27: D0926 06:01:22.474303 128492 SchedulerThread.cpp:144] SCHEDULABLE BEING REFIRED id=*NoisyCollectors* interval=1000000000 ns offset=783221187 ns now=@1378369044761823 ns fireTime=@1378369044696482 ns nextFireTime=1378370044761823 ns\ntFireTime=1378428569540704 ns\n235 ns\nns\nme=1378425746961\ncppdynamoserver: I0926 06:01:22.532336 834847 PowerReader.cpp:762] getPower succeeded! totalPower: 146.386, server power: 136.71\n\nCollection-27: D0926 06:01:22.535629 128492 SchedulerThread.cpp:144] SCHEDULABLE BEING REFIRED id=fb303:fg_policy:devbig042.lla2:::1:9176 interval=60000000000 ns offset=23946121169 ns now=@1378369106087589 ns fireTime=@1378369106021796 ns nextFireTime=1378429106021796 ns\n4 ns\n235 ns\nns\nme=1378425746961\n"})}),"\n",(0,r.jsx)(s.p,{children:"The above script usually produces interesting output on a busy production system running lots of different applications and scripts."}),"\n",(0,r.jsxs)(s.p,{children:[(0,r.jsx)(s.strong,{children:"NOTE"}),": With a vanilla install then this script may produce nothing as there isn't much actually running on such a system! If that's the case for you, in another terminal execute regular commands with bogus arguments which will generate output to the terminals stderr (e.g., ",(0,r.jsx)(s.code,{children:"'ls /unknown'"}),", ",(0,r.jsx)(s.code,{children:"'man nonsense'"}),", ",(0,r.jsx)(s.code,{children:"'cat flap'"}),", ...)."]}),"\n",(0,r.jsx)(s.p,{children:"(XXXXXXX fix echo of input on lab vm!)."}),"\n",(0,r.jsx)(s.p,{children:"Things to note:"}),"\n",(0,r.jsxs)(s.ul,{children:["\n",(0,r.jsxs)(s.li,{children:["Recent builds of bpftrace now have Big String support which supports working with strings up to 32KB in size. The default size is still 64 bytes and therefore ",(0,r.jsxs)(s.a,{href:"/docs/release_023/language#max_strlen",children:["the \ud83d\udcd6 ",(0,r.jsx)(s.code,{children:"max_strlen"})," tunable parameter"]})," has been tuned to 32KB in the ",(0,r.jsx)(s.code,{children:"config"})," block."]}),"\n",(0,r.jsxs)(s.li,{children:[(0,r.jsx)(s.code,{children:"char *"}),"'s must be explicitly converted to strings using ",(0,r.jsxs)(s.a,{href:"/docs/release_023/stdlib#str",children:["the \ud83d\udcd6 ",(0,r.jsx)(s.code,{children:"str()"})," builtin"]}),"."]}),"\n",(0,r.jsxs)(s.li,{children:[(0,r.jsxs)(s.a,{href:"/docs/release_023/stdl

1ib#builtins",children:["The \ud83d\udcd6 ",(0,r.jsx)(s.code,{children:"comm"})," builtin"]})," gives us the name of the thread doing the write call."]}),"\n"]}),"\n",(0,r.jsx)(s.h2,{id:"syscall-return-probes",children:"Syscall return probes"}),"\n",(0,r.jsxs)(s.p,{children:["As with any C function, we only have a single return value from a syscall. As an exercise, compare the return codes specified in the man pages with the output of ",(0,r.jsx)(s.code,{children:"bpftrace -lv"})," for the following syscalls exit probes:"]}),"\n",(0,r.jsxs)(s.ul,{children:["\n",(0,r.jsx)(s.li,{children:(0,r.jsx)(s.code,{children:"close"})}),"\n",(0,r.jsx)(s.li,{children:(0,r.jsx)(s.code,{children:"mmap"})}),"\n",(0,r.jsx)(s.li,{children:(0,r.jsx)(s.code,{children:"write"})}),"\n"]}),"\n",(0,r.jsx)(s.p,{children:"As you can see, the types don't agree as bpftrace always reports the return type as 'long'. We may need to cast the return value accordingly to the correct return type as specified by the man page."}),"\n",(0,r.jsx)(s.hr,{}),"\n",(0,r.jsx)(s.h2,{id:"exercises",children:"Exercises"}),"\n",(0,r.jsxs)(s.p,{children:["NOTE: before attempting the tasks in this section select the ",(0,r.jsx)(s.code,{children:"syscalls"})," option from the ",(0,r.jsx)(s.code,{children:"bpfhol"})," menu."]}),"\n",(0,r.jsxs)(s.h3,{id:"exercises-22-mmap2",children:["Exercises 2.2: ",(0,r.jsx)(s.code,{children:"mmap(2)"})]}),"\n",(0,r.jsxs)(s.ol,{children:["\n",(0,r.jsx)(s.li,{children:"Locate the process doing the most mmap calls in a 30 second period (it should be obvious :-) )."}),"\n",(0,r.jsx)(s.li,{children:"What are the sizes of the segments being created?"}),"\n",(0,r.jsxs)(s.li,{children:["Keep a count of created mappings and use that to discover the ratio of private to shared mappings (HINT: use the ",(0,r.jsx)(s.code,{children:"MAP_SHARED"})," and ",(0,r.jsx)(s.code,{children:"MAP_PRIVATE"})," flags)."]}),"\n"]}),"\n",(0,r.jsxs)(s.h3,{id:"exercises-23-open2openat2",children:["Exercises 2.3: ",(0,r.jsx)(s.code,{children:"open(2)/openat(2)"})]}),"\n",(0,r.jsxs)(s.ol,{children:["\n",(0,r.jsxs)(s.li,{children:["Write a script to show which files are being opened. (use ",(0,r.jsx)(s.code,{children:"open(2)"})," only)."]}),"\n",(0,r.jsxs)(s.li,{children:["Extend that script to show which processes are opening which file. (use ",(0,r.jsx)(s.code,{children:"open(2)"})," only)."]}),"\n",(0,r.jsxs)(s.li,{children:["Change that script to only show open calls that are creating temp files (hint: use the ",(0,r.jsx)(s.code,{children:"O_TMPFILE"})," flag with ",(0,r.jsx)(s.code,{children:"openat(2)"}),")."]}),"\n"]}),"\n",(0,r.jsxs)(s.h3,{id:"exercises-24-close2",children:["Exercises 2.4: ",(0,r.jsx)(s.code,{children:"close(2)"})]}),"\n",(0,r.jsxs)(s.ol,{children:["\n",(0,r.jsxs)(s.li,{children:["Find all ",(0,r.jsx)(s.code,{children:"close(2)"})," system calls that were passed invalid file descriptors."]}),"\n"]}),"\n",(0,r.jsxs)(s.p,{children:["In the next section we take a dive into tracing ",(0,r.jsx)(s.a,{href:"./kernel-probes",children:"kernel functions"}),"."]}),"\n",(0,r.jsx)(s.h2,{id:"further-reading",children:"Further Reading"}),"\n",(0,r.jsxs)(s.ul,{children:["\n",(0,r.jsx)(s.li,{children:(0,r.jsx)(s.a,{href:"https://www.kernel.org/doc/html/latest/process/adding-syscalls.html",children:"Adding a New System Call"})}),"\n"]}),"\n",(0,r.jsx)(s.h2,{id:"back-to-hol-intro",children:(0,r.jsx)(s.a,{href:"./intro",children:"Back to HOL Intro"})})]})}function d(e={}){const{wrapper:s}={...(0,i.R)(),...e.components};return s?(0,r.jsx)(s,{...e,children:(0,r.jsx)(h,{...e})}):h(e)}},8453:(e,s,n)=>{n.d(s,{R:()=>l,x:()=>c});var t=n(6540);const r={},i=t.createContext(r);function l(e){const s=t.useContext(i);return t.useMemo((function(){return"function"==typeof e?e(s):{...s,...e}}),[s,e])}function c(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:l(e.components),t.createElement(i.Provider,{value:s},e.children)}}}]);