1"use strict";(globalThis.webpackChunkbacalhau_website||=[]).push([[1528],{5312(e,n,i){i.r(n),i.d(n,{assets:()=>d,contentTitle:()=>r,default:()=>l,frontMatter:()=>s,metadata:()=>o,toc:()=>u});const o=JSON.parse('{"id":"guides/queueing","title":"Queueing","description":"Configure how long Bacalhau jobs wait for compute nodes that satisfy capacity and placement constraints.","source":"@site/docs/guides/queueing.md","sourceDirName":"guides","slug":"/guides/queueing","permalink":"/docs/guides/queueing","draft":false,"unlisted":false,"editUrl":"https://github.com/bacalhau-project/docs/tree/main/docs/guides/queueing.md","tags":[],"version":"current","lastUpdatedAt":1785514175000,"frontMatter":{"description":"Configure how long Bacalhau jobs wait for compute nodes that satisfy capacity and placement constraints."},"sidebar":"docs","previous":{"title":"Publishing Results","permalink":"/docs/guides/publishing-results"},"next":{"title":"Resource Requirements","permalink":"/docs/guides/resource-requirements"}}');var a=i(4848),t=i(8453);const s={description:"Configure how long Bacalhau jobs wait for compute nodes that satisfy capacity and placement constraints."},r="Queueing",d={},u=[{value:"Introduction",id:"introduction",level:2},{value:"Configuring Job Queueing",id:"configuring-job-queueing",level:2},{value:"How It Works",id:"how-it-works",level:2},{value:"Orchestrator Node",id:"orchestrator-node",level:3},{value:"Job Specification",id:"job-specification",level:3},{value:"CLI Command",id:"cli-command",level:3},{value:"Monitoring Queued Jobs",id:"monitoring-queued-jobs",level:2}];function c(e){const n={a:"a",admonition:"admonition",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",p:"p",pre:"pre",...(0,t.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.header,{children:(0,a.jsx)(n.h1,{id:"queueing",children:"Queueing"})}),"\n",(0,a.jsx)(n.h2,{id:"introduction",children:"Introduction"}),"\n",(0,a.jsx)(n.p,{children:"Job Queueing allows Bacalhau to handle situations when there are no suitable nodes available to execute a job. You can configure a time period during which the job will wait for suitable nodes to become available or free up. This feature enables better flexibility and reliability in managing your distributed workloads."}),"\n",(0,a.jsx)(n.h2,{id:"configuring-job-queueing",children:"Configuring Job Queueing"}),"\n",(0,a.jsxs)(n.p,{children:["The job queueing feature is not automatically enabled and needs to be explicitly set in your ",(0,a.jsx)(n.a,{href:"/docs/specifications/job/",children:"Job specification"})," or node configuration using the ",(0,a.jsx)(n.code,{children:"QueueTimeout"})," parameter. This parameter activates the queueing feature and defines the amount of time your job should wait for available nodes."]}),"\n",(0,a.jsx)(n.p,{children:"Node availability is determined by capacity as well as job constraints such as label selectors, engines, or publishers. For example, jobs will be queued if all nodes are currently busy, or if idle nodes do not match parameters in your job specification."}),"\n",(0,a.jsx)(n.admonition,{type:"info",children:(0,a.jsxs)(n.p,{children:["Bacalhau compute nodes regularly update their ",(0,a.jsx)(n.a,{href:"/docs/references/operators/node-management",children:"node, resource and health information"})," every 30 seconds to the orchestrator nodes. During this update period, multiple jobs may be allocated to a node, potentially exceeding its immediate available capacity. A local job queue is created at the compute node, efficiently handling the high demand as resources become available over time."]})}),"\n",(0,a.jsx)(n.h2,{id:"how-it-works",children:"How It Works"}),"\n",(0,a.jsxs)(n.p,{children:["You can set default queueing behavior for all jobs by defining the ",(0,a.jsx)(n.code,{children:"QueueTimeout"})," parameter in the node's configuration file. Alternatively, within the job specification, you can include the ",(0,a.jsx)(n.code,{children:"QueueTimeout"})," parameter directly in the configuration YAML. This flexibility allows you to tailor the queueing behavior to meet the specific needs of your distributed computing environment."]}),"\n",(0,a.jsx)(n.h3,{id:"orchestrator-node",children:"Orchestrator Node"}),"\n",(0,a.jsx)(n.p,{children:"Here's an example orchestrator node configuration that sets the default job queueing retry interval for an hour:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yaml",children:"Orchestrator:\n Scheduler:\n QueueBackoff: 1h\n"})}),"\n",(0,a.jsx)(n.admonition,{type:"info",children:(0,a.jsxs)(n.p,{children:["The ",(0,a.jsx)(n.code,{children:"QueueBackoff"})," parameter determines how frequently the scheduler will retry assigning queued jobs to compute nodes."]})}),"\n",(0,a.jsx)(n.h3,{id:"job-specification",children:"Job Specification"}),"\n",(0,a.jsxs)(n.p,{children:["Here's a sample job specification setting the ",(0,a.jsx)(n.code,{children:"QueueTimeout"})," for this specific job, overriding any node defaults:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yaml",children:"Type: batch\nCount: 1\nTasks:\n - Name: main\n Engine:\n Type: docker\n Params:\n Image: ubuntu:latest\n Entrypoint:\n - /bin/bash\n Parameters:\n - -c\n - sleep 90\n Timeouts:\n QueueTimeout: 1800\n"})}),"\n",(0,a.jsx)(n.h3,{id:"cli-command",children:"CLI Command"}),"\n",(0,a.jsxs)(n.p,{children:["You can also define timeouts for your jobs directly through the CLI using the ",(0,a.jsx)(n.code,{children:"--queue-timeout"})," flag. This provides a convenient way to specify queueing behavior on a per-job basis without modifying configuration files:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"bacalhau docker run ubuntu sleep 90 --queue-timeout 1800\n"})}),"\n",(0,a.jsx)(n.admonition,{type:"warning",children:(0,a.jsxs)(n.p,{children:["Timeouts in Bacalhau are generally governed by the ",(0,a.jsx)(n.code,{children:"TotalTimeout"})," value for your YAML specifications and the ",(0,a.jsx)(n.code,{children:"--timeout"})," flag for your CLI commands. The default total timeout value is 30 minutes. Setting a queue timeout larger than the total timeout without increasing the total timeout value will result in a validation error."]})}),"\n",(0,a.jsx)(n.h2,{id:"monitoring-queued-jobs",children:"Monitoring Queued Jobs"}),"\n",(0,a.jsx)(n.p,{children:"Jobs will be queued when all available nodes are busy or when there is no node that matches your job specifications. Let's look at how you can monitor queued jobs."}),"\n",(0,a.jsxs)(n.p,{children:["Queued jobs will initially display the ",(0,a.jsx)(n.code,{children:"Queued"})," status. Using the ",(0,a.jsx)(n.code,{children:"bacalhau job describe"})," command will show both the state of the job and the reason behind queueing."]}),"\n",(0,a.jsx)(n.p,{children:"For busy nodes:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"ID = j-d740ba46-b135-4161-bd79-795c94d215b0\nName = j-d740ba46-b135-4161-bd79-795c94d215b0\nNamespace = default\nType = batch\nState = Queued\nMessage = Job queued. not enough nodes to run job. requested: 1, available: 3, suitable: 0.\n\u2022 Node n-b75224b7: node busy with available capacity {CPU: 0.2, Memory: 12 GB, Disk: 79 GB, GPU: 0}
1\n and queue capacity {CPU: 2, Memory: 4.0 GB, Disk: 0 B, GPU: 0}\n\u2022 Node n-d42422fd: node busy with available capacity {CPU: 0.2, Memory: 12 GB, Disk: 83 GB, GPU: 0}\n and queue capacity {CPU: 3, Memory: 1.0 GB, Disk: 0 B, GPU: 0}\n\u2022 Node n-f50db1f9: node busy with available capacity {CPU: 0.2, Memory: 12 GB, Disk: 83 GB, GPU: 0}\n"})}),"\n",(0,a.jsx)(n.p,{children:"For no matching nodes:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"ID = j-0dda82b7-ad5a-4b96-b675-728c5f54f4c9\nName = j-0dda82b7-ad5a-4b96-b675-728c5f54f4c9\nNamespace = default\nType = batch\nState = Queued\nMessage = Job queued. not enough nodes to run job. requested: 1, available: 4, suitable: 0.\n\u2022 3 of 4 nodes: labels map[Architecture:amd64 Operating-System:linux owner:bacalhau]\n don't match required selectors [name = walid]\n\u2022 Node Qma5yQAk: labels map[Architecture:amd64 GPU-0:Tesla-T4 GPU-0-Memory:15360-MiB\n Operating-System:linux owner:bacalhau] don't match required selectors [name = walid]\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Once appropriate node resources become available, these jobs will transition to either a ",(0,a.jsx)(n.code,{children:"Running"})," or ",(0,a.jsx)(n.code,{children:"Completed"})," status, allowing more jobs to be assigned to matching nodes."]}),"\n",(0,a.jsx)(n.p,{children:"Here's an example of a job's lifecycle from queued to completed:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"ID = j-0dda82b7-ad5a-4b96-b675-728c5f54f4c9\nName = j-0dda82b7-ad5a-4b96-b675-728c5f54f4c9\nNamespace = default\nType = batch\nState = Completed\nCount = 1\nCreated Time = 2024-06-24 13:36:40\nModified Time = 2024-06-24 13:41:40\nVersion = 0\n\n\nSummary\nCompleted = 1\n\n\nJob History\n TIME REV. STATE TOPIC EVENT\n 2024-06-24 13:36:40 1 Pending Submission Job submitted\n 2024-06-24 13:36:40 2 Queued Queueing Job queued. not enough nodes to run job. requested: 1,\n available: 4, suitable: 0.\n \u2022 3 of 4 nodes: labels map[Architecture:amd64\n Operating-System:linux owner:bacalhau]\n don't match required selectors [name = walid]\n \u2022 Node Qma5yQAk: labels map[Architecture:amd64\n GPU-0:Tesla-T4 GPU-0-Memory:15360-MiB\n Operating-System:linux owner:bacalhau]\n don't match required selectors [name = walid]\n 2024-06-24 13:39:40 3 Running\n 2024-06-24 13:41:40 4 Completed\n\n\nExecutions\n ID NODE ID STATE DESIRED REV. CREATED MODIFIED COMMENT\n e-88cb1c72 n-73426e31 Completed Stopped 6 6m5s ago 4m4s ago Accepted job\n\n\nExecution e-88cb1c72 History\n TIME REV. STATE TOPIC EVENT\n 2024-06-24 13:39:40 1 New\n 2024-06-24 13:39:40 2 AskForBid\n 2024-06-24 15:39:40 3 AskForBidAccepted Requesting Node Accepted job\n 2024-06-24 13:39:40 4 AskForBidAccepted\n 2024-06-24 13:39:40 5 BidAccepted\n 2024-06-24 13:41:40 6 Completed\n"})})]})}function l(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},8453(e,n,i){i.d(n,{R:()=>s,x:()=>r});var o=i(6540);const a={},t=o.createContext(a);function s(e){const n=o.useContext(t);return o.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:s(e.components),o.createElement(t.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.