https://www.abhik.ai/_next/static/chunks/pages/concepts/language-i…/dataloader-pipeline-2acc9d4383872354.js

1(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[99376],{7440:(e,s,n)=>{"use strict";n.r(s),n.d(s,{BottleneckDiagnostic:()=>x,DataLoaderMemoryAnalyzer:()=>u,DataLoaderPipelineViz:()=>h,WorkerThroughputProfiler:()=>m,__N_SSG:()=>p,default:()=>g,meta:()=>j});var a=n(37876),c=n(91668),t=n(35674),l=n(26286),i=n(18847),o=n.n(i),r=n(40028);n(48230);var d=n(48645),p=!0;let h=o()(()=>n.e(9351).then(n.bind(n,9351)),{loadableGenerated:{webpack:()=>[9351]},ssr:!1,loading:r.A}),m=o()(()=>n.e(93644).then(n.bind(n,93644)),{loadableGenerated:{webpack:()=>[93644]},ssr:!1,loading:r.A}),u=o()(()=>n.e(46945).then(n.bind(n,46945)),{loadableGenerated:{webpack:()=>[46945]},ssr:!1,loading:r.A}),x=o()(()=>n.e(16315).then(n.bind(n,16315)),{loadableGenerated:{webpack:()=>[16315]},ssr:!1,loading:r.A}),j={title:"PyTorch DataLoader Pipeline",description:"PyTorch DataLoader deep dive — Dataset, Sampler, Workers, Collate internals, num_workers throughput profiling, memory analysis, serialization costs, production patterns (LMDB, WebDataset), and bottleneck diagnosis.",category:"language-internals",date:"2024-12-31",slug:"language-internals/dataloader-pipeline",order:1,tags:["pytorch","dataloader","data-pipeline","deep-learning","gpu"],relatedConcepts:["num-workers","pin-memory","data-parallel","cuda-context","unified-memory"],readingTime:"22 min",difficulty:"intermediate",keywords:["PyTorch DataLoader","num_workers optimization","pin_memory","prefetch_factor","DataLoader bottleneck","GPU data loading","persistent_workers","WebDataset LMDB","DataLoader OOM","DistributedSampler","DataLoader profiling","collate_fn custom","IterableDataset"]},k=e=>(0,a.jsx)(t.b,Object.assign({meta:j},e));function N(e){let s=Object.assign({h2:"h2",a:"a",p:"p",strong:"strong",h3:"h3",code:"code",pre:"pre",span:"span",ul:"ul",li:"li",em:"em",ol:"ol"},(0,c.RP)(),e.components),{RelatedConceptsGrid:n}=s;return n||function(e,s){throw Error("Expected "+(s?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}("RelatedConceptsGrid",!0),(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(s.h2,{id:"pytorch-dataloader-pipeline",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#pytorch-dataloader-pipeline",children:"PyTorch DataLoader Pipeline"})}),"\n",(0,a.jsx)(s.h2,{id:"why-the-dataloader-matters",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#why-the-dataloader-matters",children:"Why the DataLoader Matters"})}),"\n",(0,a.jsx)(s.p,{children:"The DataLoader is the bridge between your data on disk and your model on GPU. If the GPU finishes a batch before the next one arrives, it sits idle — burning electricity and time. Most ML engineers lose 30–50% of potential training throughput to DataLoader misconfiguration. Understanding the pipeline internals turns guessing into engineering."}),"\n",(0,a.jsx)(s.h2,{id:"pipeline-visualization",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#pipeline-visualization",children:"Pipeline Visualization"})}),"\n",(0,a.jsx)(h,{}),"\n",(0,a.jsx)(s.h2,{id:"the-four-components",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#the-four-components",children:"The Four Components"})}),"\n",(0,a.jsxs)(s.p,{children:["Every DataLoader is composed of four cooperating components: a ",(0,a.jsx)(s.strong,{children:"Dataset"})," that knows how to load a single sample, a ",(0,a.jsx)(s.strong,{children:"Sampler"})," that decides the order, ",(0,a.jsx)(s.strong,{children:"Workers"})," that load in parallel, and a ",(0,a.jsx)(s.strong,{children:"Collate"})," function that assembles individual samples into batches. Getting each one right is the difference between a GPU that trains and a GPU that waits."]}),"\n",(0,a.jsx)(s.h3,{id:"1-dataset",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#1-dataset",children:"1. Dataset"})}),"\n",(0,a.jsxs)(s.p,{children:["The ",(0,a.jsx)(s.code,{children:"Dataset"})," class defines how to access individual samples. The critical design decision is ",(0,a.jsxs)(s.strong,{children:["what you store in ",(0,a.jsx)(s.code,{children:"__init__"})]})," versus what you load in ",(0,a.jsx)(s.code,{children:"__getitem__"}),":"]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"from"})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"utils",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"data ",(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," Dataset\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"from"})," pathlib ",(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," Path\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"from"})," PIL ",(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," Image\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"class"})," ",(0,a.jsx)(s.span,{className:"token class-name",children:"ImageDataset"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"Dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"__init__"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," root_dir",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," transform",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"None"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"paths ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," ",(0,a.jsx)(s.span,{className:"token builtin",children:"sorted"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"Path",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"root_dir",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"glob",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:"'**/*.jpg'"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Store paths, NOT loaded images"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"transform ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," transform\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"__len__"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        ",(0,a.jsx)(s.span,{className:"token keyword",children:"return"})," ",(0,a.jsx)(s.span,{className:"token builtin",children:"len"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"paths",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"__getitem__"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," idx",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        img ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," Image",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),(0,a.jsx)(s.span,{className:"token builtin",children:"open"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"paths",(0,a.jsx)(s.span,{className:"token punctuation",children:"["}),"idx",(0,a.jsx)(s.span,{className:"token punctuation",children:"]"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"convert",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:"'RGB'"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Disk I/O here — runs in workers"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        ",(0,a.jsx)(s.span,{className:"token keyword",children:"if"})," self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"transform",(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["            img ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"transform",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"img",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        label ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"paths",(0,a.jsx)(s.span,{className:"token punctuation",children:"["}),"idx",(0,a.jsx)(s.span,{className:"token punctuation",children:"]"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"parent",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"name  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Directory name as label"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        ",(0,a.jsx)(s.span,{className:"token keyword",children:"return"})," img",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," label\n"]})]})}),"\n",(0,a.jsx)(d.Callout,{type:"warning",title:"Store Paths, Not Data",children:(0,a.jsxs)(s.p,{children:["Store file paths in ",(0,a.jsx)(s.code,{children:"__init__"}),", not loaded data. Each worker forks the process\n— if your Dataset holds 10GB of tensors in ",(0,a.jsx)(s.code,{children:"__init__"}

1),", each worker\ncopies all 10GB. With 4 workers, that’s 40GB of CPU RAM before training\neven starts."]})}),"\n",(0,a.jsx)(s.h3,{id:"2-sampler",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#2-sampler",children:"2. Sampler"})}),"\n",(0,a.jsxs)(s.p,{children:["The Sampler determines the order in which indices are fed to the Dataset. For single-GPU training, PyTorch handles this automatically. For multi-GPU DDP training, ",(0,a.jsx)(s.code,{children:"DistributedSampler"})," ensures each GPU sees a unique, non-overlapping subset:"]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"from"})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"utils",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"data ",(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," DistributedSampler\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# Single GPU"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["loader ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," shuffle",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Uses RandomSampler internally"}),"\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# Multi-GPU with DDP"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["sampler ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," DistributedSampler",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," num_replicas",(0,a.jsx)(s.span,{className:"token operator",children:"="}),"world_size",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," rank",(0,a.jsx)(s.span,{className:"token operator",children:"="}),"rank",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," shuffle",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["loader ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," sampler",(0,a.jsx)(s.span,{className:"token operator",children:"="}),"sampler",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Don't set shuffle=True when using sampler!"}),"\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# IMPORTANT: call sampler.set_epoch(epoch) each epoch for proper shuffling"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," epoch ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"token builtin",children:"range"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"num_epochs",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    sampler",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"set_epoch",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"epoch",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Without this, every epoch sees the same order"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," batch ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," loader",(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        train",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"batch",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsx)(d.Callout,{type:"note",title:"DistributedSampler Pitfall",children:(0,a.jsxs)(s.p,{children:["If you forget ",(0,a.jsx)(s.code,{children:"sampler.set_epoch(epoch)"}),", every epoch will iterate samples in\nthe same order. The sampler uses the epoch number as a random seed —\nwithout updating it, the shuffle is deterministic and identical across epochs."]})}),"\n",(0,a.jsx)(s.h3,{id:"3-workers",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#3-workers",children:"3. Workers"})}),"\n",(0,a.jsxs)(s.p,{children:["Worker processes load data in parallel using Python’s ",(0,a.jsx)(s.code,{children:"multiprocessing"}),", bypassing the GIL entirely. The key parameter most engineers overlook is ",(0,a.jsx)(s.code,{children:"prefetch_factor"}),":"]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:["DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    num_workers",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token number",children:"4"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    prefetch_factor",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token number",children:"2"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","}),"       ",(0,a.jsx)(s.span,{className:"token comment",children:"# Each worker pre-loads 2 batches ahead"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    persistent_workers",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Keep workers alive between epochs (avoid fork overhead)"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsxs)(s.p,{children:[(0,a.jsx)(s.code,{children:"prefetch_factor=2"})," means each worker pre-loads 2 batches ahead. With 4 workers and ",(0,a.jsx)(s.code,{children:"prefetch_factor=2"}),", there are up to ",(0,a.jsx)(s.strong,{children:"8 batches in flight"}),". Each batch in flight costs ",(0,a.jsx)(s.code,{children:"batch_size \xd7 sample_size"})," of CPU RAM. For ImageNet with batch_size=64 and 224\xd7224\xd73 float32 images, that’s roughly ",(0,a.jsx)(s.code,{children:"8 \xd7 64 \xd7 600KB ≈ 300MB"})," of prefetched data sitting in memory."]}),"\n",(0,a.jsx)(s.h3,{id:"4-collate-function",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#4-collate-function",children:"4. Collate Function"})}),"\n",(0,a.jsxs)(s.p,{children:["The ",(0,a.jsx)(s.code,{children:"collate_fn"})," combines individual samples into batches. The default works for uniform tensors, but real-world data often requires custom handling — especially variable-length sequences:"]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," torch\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"variable_length_collate"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"batch",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token triple-quoted-string string",children:'"""Handle sequences of different lengths by padding."""'}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    sequences",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," labels ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," ",(0,a.jsx)(s.span,{className:"token builtin",children:"zip"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token operator",children:"*"}),"batch",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    lengths ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," ",(0,a.jsx)(s.span,{className:"token punctuation",children:"["}),(0,a.jsx)(s.span,{className:"token builtin",children:"len"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"s",(0,a.jsx)(s.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," s ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," sequences",(0,a.jsx)(s.span,{className:"token punctuation",children:"]"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    padded ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"nn",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"utils",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"rnn",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"pad_sequence",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"sequences",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," batch_first",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"return"})," padded",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"tensor",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"labels",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"tensor",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"lengths",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["loader ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," collate_fn",(0,a.jsx)(s.span,{className:"token operator",children:"="}),"variable_length_collate",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsxs)(s.p,{children:[(0,a.jsx)(s.strong,{children:"Important"}),": Collate runs in the ",(0,a.jsx)(s.strong,{children:"main process"}

1),", not in workers. If your collate function is expensive (complex padding, dynamic batching), it becomes a serialization bottleneck."]}),"\n",(0,a.jsx)(s.h2,{id:"worker-throughput-profiling",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#worker-throughput-profiling",children:"Worker Throughput Profiling"})}),"\n",(0,a.jsx)(m,{}),"\n",(0,a.jsxs)(s.p,{children:["The optimal ",(0,a.jsx)(s.code,{children:"num_workers"})," depends on your ",(0,a.jsx)(s.strong,{children:"dataset"}),", not your hardware. For ImageNet (JPEG decode), 4–8 workers saturate most SSDs. For tabular data loaded from CSV or Parquet, 0–2 workers is often best because pickle serialization between worker and main process costs more than the loading itself. The profiler above lets you see this tradeoff directly."]}),"\n",(0,a.jsx)(s.h2,{id:"the-serialization-tax",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#the-serialization-tax",children:"The Serialization Tax"})}),"\n",(0,a.jsxs)(s.p,{children:["Every sample loaded by a worker must be serialized (pickled) through a ",(0,a.jsx)(s.code,{children:"multiprocessing.Queue"})," to reach the main process. For a 224\xd7224\xd73 float32 image, that’s ~600KB per sample. With ",(0,a.jsx)(s.code,{children:"batch_size=32"})," and 4 workers, the main process deserializes ~75MB per batch. This is why ",(0,a.jsx)(s.code,{children:"num_workers=0"})," sometimes beats ",(0,a.jsx)(s.code,{children:"num_workers=4"})," for small datasets."]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# Measure serialization overhead"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," pickle",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," time\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["sample ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:"["}),(0,a.jsx)(s.span,{className:"token number",children:"0"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"]"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["start ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," time",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"time",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," _ ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"token builtin",children:"range"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token number",children:"1000"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    pickle",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"dumps",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"sample",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"print"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsxs)(s.span,{className:"token string-interpolation",children:[(0,a.jsx)(s.span,{className:"token string",children:'f"Serialization: '}),(0,a.jsxs)(s.span,{className:"token interpolation",children:[(0,a.jsx)(s.span,{className:"token punctuation",children:"{"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"time",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"time",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token operator",children:"-"}),"start",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token operator",children:"/"}),(0,a.jsx)(s.span,{className:"token number",children:"1000"}),(0,a.jsx)(s.span,{className:"token operator",children:"*"}),(0,a.jsx)(s.span,{className:"token number",children:"1000"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),(0,a.jsx)(s.span,{className:"token format-spec",children:".2f"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"}"})]}),(0,a.jsx)(s.span,{className:"token string",children:' ms per sample"'})]}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsx)(s.p,{children:"If serialization takes longer than loading, workers are doing more harm than good. Common solutions:"}),"\n",(0,a.jsxs)(s.ul,{children:["\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"Return numpy arrays instead of PIL Images"})," — numpy arrays serialize much faster via shared memory"]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsxs)(s.strong,{children:["Use ",(0,a.jsx)(s.code,{children:"num_workers=0"})]})," for datasets that fit in RAM and load quickly"]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsxs)(s.strong,{children:["Switch to ",(0,a.jsx)(s.code,{children:"IterableDataset"})]})," with shared-memory backing for large-scale streaming"]}),"\n"]}),"\n",(0,a.jsx)(s.h2,{id:"memory-analysis",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#memory-analysis",children:"Memory Analysis"})}),"\n",(0,a.jsx)(u,{}),"\n",(0,a.jsxs)(s.p,{children:["When ",(0,a.jsx)(s.code,{children:"num_workers > 0"}),", the DataLoader forks the main process. On Linux, ",(0,a.jsx)(s.code,{children:"fork()"})," uses ",(0,a.jsx)(s.strong,{children:"copy-on-write"})," (CoW) semantics — child processes share the parent’s memory pages until either process writes to them. This means your Dataset’s ",(0,a.jsx)(s.code,{children:"__init__"})," data is shared across workers ",(0,a.jsx)(s.em,{children:"as long as workers don’t modify it"}),". The moment a worker writes to a shared page, the OS copies the entire 4KB page. NumPy arrays with reference counting trigger C

1oW on every access because Python updates the refcount."]}),"\n",(0,a.jsx)(s.h2,{id:"pin_memory-and-gpu-transfer",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#pin_memory-and-gpu-transfer",children:"pin_memory and GPU Transfer"})}),"\n",(0,a.jsxs)(s.p,{children:[(0,a.jsx)(s.code,{children:"pin_memory=True"})," allocates tensors in page-locked (pinned) CPU memory. The GPU can DMA directly from pinned memory without the CPU’s involvement — roughly 2x faster than regular pageable memory transfer. The cost: pinned memory can’t be swapped to disk, so it reduces available RAM."]}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# Without pin_memory: CPU copies to pinned staging area, then DMA to GPU"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# With pin_memory: data is already pinned, DMA directly"}),"\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["loader ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," DataLoader",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"dataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," pin_memory",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Always use for GPU training"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," batch ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," loader",(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    inputs ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," batch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"to",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:"'cuda'"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","})," non_blocking",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# non_blocking with pin_memory = async transfer"}),"\n"]})]})}),"\n",(0,a.jsxs)(s.p,{children:["The combination of ",(0,a.jsx)(s.code,{children:"pin_memory=True"})," and ",(0,a.jsx)(s.code,{children:"non_blocking=True"})," allows the CPU to continue preparing the next batch while the current batch transfers to the GPU asynchronously. Without both flags, the ",(0,a.jsx)(s.code,{children:".to('cuda')"})," call blocks until the transfer completes."]}),"\n",(0,a.jsx)(s.h2,{id:"production-patterns",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#production-patterns",children:"Production Patterns"})}),"\n",(0,a.jsx)(s.p,{children:"For production training at scale, the standard Dataset + per-file loading pattern breaks down. Here are the three main alternatives:"}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# LMDB — memory-mapped database, eliminates per-file overhead"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," lmdb\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["env ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," lmdb",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),(0,a.jsx)(s.span,{className:"token builtin",children:"open"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:"'dataset.lmdb'"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","})," readonly",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"True"}),(0,a.jsx)(s.span,{className:"token punctuation",children:","})," lock",(0,a.jsx)(s.span,{className:"token operator",children:"="}),(0,a.jsx)(s.span,{className:"token boolean",children:"False"}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"with"})," env",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"begin",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"})," ",(0,a.jsx)(s.span,{className:"token keyword",children:"as"})," txn",(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    data ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," txn",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"get",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}

1),"key",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"  ",(0,a.jsx)(s.span,{className:"token comment",children:"# Memory-mapped read — no file open/close"}),"\n"]})]})}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# WebDataset — sequential tar reads, excellent for cloud storage"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," webdataset ",(0,a.jsx)(s.span,{className:"token keyword",children:"as"})," wds\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["dataset ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," ",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    wds",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"WebDataset",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:'"s3://bucket/shard-{000..099}.tar"'}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"decode",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:'"pil"'}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"to_tuple",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token string",children:'"jpg"'}),(0,a.jsx)(s.span,{className:"token punctuation",children:","})," ",(0,a.jsx)(s.span,{className:"token string",children:'"cls"'}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsx)(s.pre,{className:"language-python",children:(0,a.jsxs)(s.code,{className:"language-python code-highlight",children:[(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token comment",children:"# IterableDataset — streaming without random access"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"from"})," torch",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"utils",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"data ",(0,a.jsx)(s.span,{className:"token keyword",children:"import"})," IterableDataset",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," get_worker_info\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:[(0,a.jsx)(s.span,{className:"token keyword",children:"class"})," ",(0,a.jsx)(s.span,{className:"token class-name",children:"StreamingDataset"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"IterableDataset",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"__init__"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:","})," data_source",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"data_source ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," data_source\n"]}),(0,a.jsx)(s.span,{className:"code-line",children:"\n"}),(0,a.jsxs)(s.span,{className:"code-line",children:["    ",(0,a.jsx)(s.span,{className:"token keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"token function",children:"__iter__"}),(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"self",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        worker_info ",(0,a.jsx)(s.span,{className:"token operator",children:"="})," get_worker_info",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        ",(0,a.jsx)(s.span,{className:"token comment",children:"# Split work across workers to avoid duplicates"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["        ",(0,a.jsx)(s.span,{className:"token keyword",children:"for"})," sample ",(0,a.jsx)(s.span,{className:"token keyword",children:"in"})," self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"stream_for_worker",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"worker_info",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),(0,a.jsx)(s.span,{className:"token punctuation",children:":"}),"\n"]}),(0,a.jsxs)(s.span,{className:"code-line",children:["            ",(0,a.jsx)(s.span,{className:"token keyword",children:"yield"})," self",(0,a.jsx)(s.span,{className:"token punctuation",children:"."}),"transform",(0,a.jsx)(s.span,{className:"token punctuation",children:"("}),"sample",(0,a.jsx)(s.span,{className:"token punctuation",children:")"}),"\n"]})]})}),"\n",(0,a.jsx)(s.p,{children:(0,a.jsx)(s.strong,{children:"When to use each:"})}),"\n",(0,a.jsxs)(s.ul,{children:["\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"LMDB"}),": Random access needed, single-machine training, dataset fits on local disk"]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"WebDataset"}),": Cloud storage (S3/GCS), large-scale distributed training, sequential access is acceptable"]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"IterableDataset"}),": Streaming from databases, message queues, or infinite data sources"]}),"\n"]}),"\n",(0,a.jsx)(s.h2,{id:"troubleshooting-diagnosing-bottlenecks",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#troubleshooting-diagnosing-bottlenecks",children:"Troubleshooting: Diagnosing Bottlenecks"})}),"\n",(0,a.jsx)(x,{}),"\n",(0,a.jsx)(l.e,{headingLevel:2,children:(0,a.jsx)("div",{className:"my-8 rounded-2xl bg-gradient-to-br from-teal-50 to-cyan-50 p-6 dark:from-teal-950/30 dark:to-cyan-950/30 dark:ring-1 dark:ring-teal-800/50",children:(0,a.jsxs)(s.ol,{children:["\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"num_workers depen

1ds on your dataset, not your GPU"})," — JPEG decode needs 4–8 workers; CSV needs 0–2."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsxs)(s.strong,{children:["Store paths, not data, in ",(0,a.jsx)(s.code,{children:"__init__"})]})," — workers fork the process. In-memory datasets get copied per worker."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsxs)(s.strong,{children:[(0,a.jsx)(s.code,{children:"pin_memory=True"})," + ",(0,a.jsx)(s.code,{children:"non_blocking=True"})]})," — 2x faster CPU→GPU transfer via DMA."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"prefetch_factor controls memory"})," — ",(0,a.jsx)(s.code,{children:"prefetch_factor \xd7 num_workers \xd7 batch_size \xd7 sample_size"})," = RAM used."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"Serialization is the hidden cost"})," — pickle overhead makes ",(0,a.jsx)(s.code,{children:"num_workers=0"})," faster for small/fast datasets."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsxs)(s.strong,{children:["Use ",(0,a.jsx)(s.code,{children:"DistributedSampler.set_epoch()"})]})," — without it, every DDP epoch gets the same sample order."]}),"\n",(0,a.jsxs)(s.li,{children:[(0,a.jsx)(s.strong,{children:"Profile before optimizing"})," — ",(0,a.jsx)(s.code,{children:"torch.profiler"})," shows whether you’re data-bound or compute-bound."]}),"\n"]})})}),"\n",(0,a.jsx)(s.h2,{id:"further-reading",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#further-reading",children:"Further Reading"})}),"\n",(0,a.jsxs)(s.ul,{children:["\n",(0,a.jsx)(s.li,{children:(0,a.jsx)(s.a,{href:"https://pytorch.org/docs/stable/data.html",children:"PyTorch DataLoader Docs"})}),"\n",(0,a.jsx)(s.li,{children:(0,a.jsx)(s.a,{href:"https://github.com/webdataset/webdataset",children:"WebDataset"})}),"\n",(0,a.jsx)(s.li,{children:(0,a.jsx)(s.a,{href:"https://github.com/libffcv/ffcv",children:"FFCV: Fast Forward Computer Vision"})}),"\n",(0,a.jsx)(s.li,{children:(0,a.jsx)(s.a,{href:"https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html",children:"PyTorch Performance Tuning Guide"})}),"\n"]}),"\n",(0,a.jsx)(s.h2,{id:"related-concepts",children:(0,a.jsx)(s.a,{className:"heading-link",href:"#related-concepts",children:"Related concepts"})}),"\n",(0,a.jsx)(n,{})]})}void 0!==j&&j&&((void 0===j.wordCount||null===j.wordCount)&&(j.wordCount=1248),j.readingTime||(j.readingTime="6 min"));let g=function(e={}){return(0,a.jsx)(k,Object.assign({},e,{children:(0,a.jsx)(N,e)}))}},26286:(e,s,n)=>{"use strict";n.d(s,{e:()=>c});var a=n(37876);function c({children:e,title:s="Key Takeaways",id:n="key-takeaways",headingLevel:c=3,className:t=""}){return(0,a.jsxs)("aside",{className:["not-prose my-6 rounded-xl bg-teal-50 px-5 py-4 dark:bg-teal-900/20","[&_ol]:my-0 [&_ol]:space-y-1.5 [&_ol]:pl-5 [&_ul]:my-0 [&_ul]:space-y-1.5 [&_ul]:pl-5","[&_li]:my-0 [&_li]:text-sm [&_li]:leading-snug [&_li]:text-zinc-700 dark:[&_li]:text-zinc-300","[&_code]:text-[0.9em] [&_code]:text-teal-700 dark:[&_code]:text-teal-300","[&_p]:my-0 [&_p]:text-sm","[&_strong]:font-semibold [&_strong]:text-zinc-900 dark:[&_strong]:text-zinc-100",t].filter(Boolean).join(" "),children:[(0,a.jsx)(2===c?"h2":"h3",{id:n,className:"!mb-2.5 !mt-0 text-lg font-semibold tracking-tight text-zinc-900 dark:text-zinc-100",children:(0,a.jsx)("a",{className:"heading-link",href:`#${n}`,children:s})}),e]})}},31405:(e,s,n)=>{(window.__NEXT_P=window.__NEXT_P||[]).push(["/concepts/language-internals/dataloader-pipeline",function(){return n(7440)}])},40028:(e,s,n)=>{"use strict";n.d(s,{A:()=>c,b:()=>t});var a=n(37876);function c({className:e="h-64"}){return(0,a.jsx)("div",{className:`${e} animate-pulse rounded-xl bg-zinc-100 dark:bg-zinc-800`})}function t(){return(0,a.jsx)("div",{className:"h-48 animate-pulse rounded-xl bg-zinc-100 dark:bg-zinc-800"})}}},e=>{e.O(0,[15863,82667,35674,48645,90636,46593,38792],()=>e(e.s=31405)),_N_E=e.O()}]);