https://datamitsu.com/assets/js/8cbb6b44.fce98f91.js

1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([["9401"],{20485(e,t,r){r.r(t),r.d(t,{metadata:()=>n,default:()=>h,frontMatter:()=>o,contentTitle:()=>c,toc:()=>l,assets:()=>d});var n=JSON.parse('{"id":"guides/architecture/discovery","title":"File Discovery & Project Detection","description":"How datamitsu walks the repository tree respecting .gitignore rules and detects project boundaries for monorepo-aware tool execution","source":"@site/docs/guides/architecture/discovery.md","sourceDirName":"guides/architecture","slug":"/guides/architecture/discovery","permalink":"/docs/guides/architecture/discovery","draft":false,"unlisted":false,"editUrl":"https://github.com/datamitsu/datamitsu/tree/main/website/docs/guides/architecture/discovery.md","tags":[],"version":"current","frontMatter":{"title":"File Discovery & Project Detection","description":"How datamitsu walks the repository tree respecting .gitignore rules and detects project boundaries for monorepo-aware tool execution"},"sidebar":"docsSidebar","previous":{"title":"Parallel Execution","permalink":"/docs/guides/architecture/execution"},"next":{"title":"Caching Strategy","permalink":"/docs/guides/architecture/caching"}}'),s=r(8525),i=r(98988);let o={title:"File Discovery & Project Detection",description:"How datamitsu walks the repository tree respecting .gitignore rules and detects project boundaries for monorepo-aware tool execution"},c="File Discovery & Project Detection",d={},l=[{value:".gitignore-Aware Traversal",id:"gitignore-aware-traversal",level:2},{value:"How It Works",id:"how-it-works",level:3},{value:"Why It Matters",id:"why-it-matters",level:3},{value:"Project Auto-Detection",id:"project-auto-detection",level:2},{value:"Marker-Based System",id:"marker-based-system",level:3},{value:"How Tools Use Project Types",id:"how-tools-use-project-types",level:3},{value:"What Happens When Markers Are Missing",id:"what-happens-when-markers-are-missing",level:3},{value:"Monorepo Example",id:"monorepo-example",level:3},{value:"How Discovery Feeds the Pipeline",id:"how-discovery-feeds-the-pipeline",level:2}];function a(e){let t={a:"a",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",mermaid:"mermaid",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,i.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(t.header,{children:(0,s.jsx)(t.h1,{id:"file-discovery--project-detection",children:"File Discovery & Project Detection"})}),"\n",(0,s.jsxs)(t.p,{children:["File discovery is the first stage of datamitsu's execution pipeline. It walks the repository tree to find files that match tool glob patterns, while project detection identifies independent project boundaries within monorepos. Together they determine ",(0,s.jsx)(t.em,{children:"what"})," files exist and ",(0,s.jsx)(t.em,{children:"where"})," projects live before the ",(0,s.jsx)(t.a,{href:"/docs/guides/architecture/planner",children:"planner"})," decides what to do with them."]}),"\n",(0,s.jsx)(t.h2,{id:"gitignore-aware-traversal",children:".gitignore-Aware Traversal"}),"\n",(0,s.jsxs)(t.p,{children:["The file walker respects ",(0,s.jsx)(t.code,{children:".gitignore"})," rules at every directory level, plus the two repository-wide exclude files git reads: ",(0,s.jsx)(t.code,{children:".git/info/exclude"})," and your personal ",(0,s.jsx)(t.code,{children:"core.excludesFile"})," (by default ",(0,s.jsx)(t.code,{children:"~/.config/git/ignore"}),"). This means datamitsu never processes files your version control system ignores \u2014 ",(0,s.jsx)(t.code,{children:"node_modules/"}),", ",(0,s.jsx)(t.code,{children:"dist/"}),", ",(0,s.jsx)(t.code,{children:".venv/"}),", build artifacts, and any other ignored paths are automatically excluded."]}),"\n",(0,s.jsx)(t.h3,{id:"how-it-works",children:"How It Works"}),"\n",(0,s.jsx)(t.p,{children:"For a fix/lint/check run, the runner starts one walk at the git root and shares\nits sorted result with every downstream consumer. At each level, the walker:"}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:["Checks for a ",(0,s.jsx)(t.code,{children:".gitignore"})," file and merges its rules with the parent's rules"]}),"\n",(0,s.jsxs)(t.li,{children:["Skips ",(0,s.jsx)(t.code,{children:".git"})," directories and symlinks (avoiding circular references)"]}),"\n",(0,s.jsx)(t.li,{children:"Processes subdirectories in parallel (up to 8 concurrent goroutines)"}),"\n",(0,s.jsx)(t.li,{children:"Collects non-ignored files in a thread-safe accumulator"}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["After the parallel walk finishes, the complete absolute-path list is sorted once.\nThat deterministic inventory is reused by bundled ",(0,s.jsx)(t.code,{children:".datamitsuignore"})," validation,\nproject detection, glob matching, unit membership, and both halves of\n",(0,s.jsx)(t.code,{children:"datamitsu check"}),"."]}),"\n",(0,s.jsx)(t.mermaid,{value:'graph TD\n    R["Git Root"] --\x3e G["Collect excludes and .gitignore rules"]\n    G --\x3e W["Walk directory tree"]\n    W --\x3e C{".gitignore match?"}\n    C --\x3e|"Yes"| S["Skip file/directory"]\n    C --\x3e|"No"| D{"Directory?"}\n    D --\x3e|"Yes"| N["Enter subdirectory (parallel)"]\n    D --\x3e|"No"| F["Add to file list"]\n    N --\x3e G2["Merge child .gitignore rules"]\n    G2 --\x3e W\n\n    style R fill:#e8f4fd,stroke:#2196f3\n    style S fill:#ffebee,stroke:#f44336\n    style F fill:#e8f5e9,stroke:#4caf50'}),"\n",(0,s.jsx)(t.h3,{id:"why-it-matters",children:"Why It Matters"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"No wasted work:"})," Ignored directories like ",(0,s.jsx)(t.code,{children:"node_modules/"})," (which can contain hundreds of thousands of files) are skipped entirely, not entered and then filtered"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Correct behavior:"})," The same files your ",(0,s.jsx)(t.code,{children:"git status"})," sees are the files datamitsu processes \u2014 no surprising lint errors from generated code or vendored dependencies"]}

1),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Cascading rules:"})," A ",(0,s.jsx)(t.code,{children:".gitignore"})," in a subdirectory extends (not replaces) the parent's rules, matching how git itself handles ignore patterns"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Git's precedence:"})," ",(0,s.jsx)(t.code,{children:"core.excludesFile"})," is weakest, ",(0,s.jsx)(t.code,{children:".git/info/exclude"})," overrides it, and any ",(0,s.jsx)(t.code,{children:".gitignore"})," overrides both. In a linked worktree, ",(0,s.jsx)(t.code,{children:"info/exclude"})," is read from the main repository, as git does"]}),"\n"]}),"\n",(0,s.jsx)(t.h2,{id:"project-auto-detection",children:"Project Auto-Detection"}),"\n",(0,s.jsxs)(t.p,{children:["In monorepos with multiple packages, tools need to know which project each file belongs to. datamitsu detects projects by looking for ",(0,s.jsx)(t.strong,{children:"marker files"})," \u2014 files whose presence indicates a project root."]}),"\n",(0,s.jsx)(t.h3,{id:"marker-based-system",children:"Marker-Based System"}),"\n",(0,s.jsx)(t.p,{children:"Each project type is defined by a list of marker glob patterns:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-javascript",children:'export function getConfig(input) {\n  return {\n    ...input,\n    projectTypes: {\n      typescript: {\n        markers: ["package.json"],\n        description: "Node.js / TypeScript project",\n      },\n      golang: {\n        markers: ["go.mod"],\n        description: "Go module",\n      },\n      python: {\n        markers: ["pyproject.toml"],\n        description: "Python project",\n      },\n    },\n  };\n}\n'})}),"\n",(0,s.jsx)(t.p,{children:"The detector scans the shared file inventory and matches each path against the\nmarker patterns; it does not traverse the repository again. Every directory\ncontaining a matching marker becomes a detected project location. A single\nrepository can contain dozens of detected projects across multiple types."}),"\n",(0,s.jsx)(t.h3,{id:"how-tools-use-project-types",children:"How Tools Use Project Types"}),"\n",(0,s.jsxs)(t.p,{children:["Tools declare which project types they apply to via the ",(0,s.jsx)(t.code,{children:"projectTypes"})," field. When the planner creates tasks, it only assigns a tool to projects that match:"]}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-javascript",children:'tools: {\n  tsc: {\n    projectTypes: ["typescript"],\n    operations: {\n      lint: {\n        app: "typescript",\n        args: ["--noEmit"],\n        scope: "per-project",\n        globs: ["**/*.ts", "**/*.tsx"],\n        priority: 20,\n      },\n    },\n  },\n  "golangci-lint": {\n    projectTypes: ["golang"],\n    operations: {\n      lint: {\n        app: "golangci-lint",\n        args: ["run"],\n        scope: "per-project",\n        globs: ["**/*.go"],\n        priority: 20,\n      },\n    },\n  },\n},\n'})}),"\n",(0,s.jsx)(t.p,{children:"With this configuration in a monorepo:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"tsc"})," runs only in directories containing ",(0,s.jsx)(t.code,{children:"package.json"})]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.code,{children:"golangci-lint"})," runs only in directories containing ",(0,s.jsx)(t.code,{children:"go.mod"})]}),"\n",(0,s.jsx)(t.li,{children:"Neither tool wastes time in projects it doesn't apply to"}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"what-happens-when-markers-are-missing",children:"What Happens When Markers Are Missing"}),"\n",(0,s.jsxs)(t.p,{children:["If a tool specifies ",(0,s.jsx)(t.code,{children:'projectTypes: ["typescript"]'})," but no ",(0,s.jsx)(t.code,{children:"package.json"})," exists anywhere in the repository, that tool simply produces zero tasks. No error, no warning \u2014 the tool is silently skipped because there are no matching projects."]}),"\n",(0,s.jsx)(t.p,{children:"This is intentional: a shared configuration can define tools for many project types, and only the relevant ones activate based on what the repository actually contains."}),"\n",(0,s.jsx)(t.h3,{id:"monorepo-example",children:"Monorepo Example"}),"\n",(0,s.jsx)(t.p,{children:"Consider this repository structure:"}),"\n",(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{children:"repo/\n\u251C\u2500\u2500 packages/\n\u2502   \u251C\u2500\u2500 frontend/\n\u2502   \u2502   \u251C\u2500\u2500 package.json      \u2190 typescript project detected\n\u2502   \u2502   \u2514\u2500\u2500 src/\n\u2502   \u251C\u2500\u2500 api/\n\u2502   \u2502   \u251C\u2500\u2500 go.mod             \u2190 golang project detected\n\u2502   \u2502   \u2514\u2500\u2500 main.go\n\u2502   \u2514\u2500\u2500 shared/\n\u2502       \u251C\u2500\u2500 package.json      \u2190 typescript project detected\n\u2502       \u2514\u2500\u2500 src/\n\u2514\u2500\u2500 services/\n    \u2514\u2500\u2500 worker/\n        \u251C\u2500\u2500 pyproject.toml     \u2190 python project detected\n        \u2514\u2500\u2500 main.py\n"})}),"\n",(0,s.jsx)(t.p,{children:"datamitsu detects four projects:"}),"\n",(0,s.jsxs)(t.table,{children:[(0,s.jsx)(t.thead,{children:(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.th,{children:"Project"}),(0,s.jsx)(t.th,{children:"Type"}),(0,s.jsx)(t.th,{children:"Tools that apply"})]})}),(0,s.jsxs)(t.tbody,{children:[(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:(0,s.jsx)(t.code,{children:"packages/frontend/"})}),(0,s.jsx)(t.td,{children:"typescript"}),(0,s.jsx)(t.td,{children:"eslint, tsc, prettier"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:(0,s.jsx)(t.code,{children:"packages/shared/"})}),(0,s.jsx)(t.td,{children:"typescript"}),(0,s.jsx)(t.td,{children:"eslint, tsc, prettier"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:(0,s.jsx)(t.code,{children:"packages/api/"})}),(0,s.jsx)(t.td,{children:"golang"}),(0,s.jsx)(t.td,{children:"golangci-lint"})]}),(0,s.jsxs)(t.tr,{children:[(0,s.jsx)(t.td,{children:(0,s.jsx)(t.code,{children:"services/worker/"})}),(0,s.jsx)(t.td,{children:"python"}),(0,s.jsx)(t.td,{children:"ruff, mypy"})]})]})]}),"\n",(0,s.jsxs)(t.p,{children:["Each tool runs independently in its project directory with isolated cache paths \u2014 see ",(0,s.jsx)(t.a,{href:"/docs/guides/architecture/caching",children:"Caching Strategy"})," for details on per-project cache isolation."]}),"\n",(0,s.jsx)(t.h2,{id:"how-discovery-feeds-the-pipeline",children:"How Discovery Feeds the Pipeline"}),"\n",(0,s.jsxs)(t.p,{children:["The output of file discovery and project detection flows directly into the ",(0,s.jsx)(t.a,{href:"/docs/guides/architecture/planner",children:"planner"}),":"]}),"\n",(0,s.jsxs)(t.ol,{children:["\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"File list"})," \u2014 all non-ignored files in the repository, used to match tool glob patterns"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"Project locations"})," \u2014 detected project directories and their types, used to scope per-project tools"]}),"\n",(0,s.jsxs)(t.li,{children:[(0,s.jsx)(t.strong,{children:"CWD restriction"})," \u2014 when running from a subdirectory, both file list and project locations are filtered to the current subtree (see ",(0,s.jsx)(t.a,{href:"/docs/guides/architecture/planner#cwd-subtree-restriction",children:"CWD-Subtree Restriction"}),")"]}),"\n"]}),"\n",(0,s.jsxs)(t.p,{children:["The expensive tree traversal is centralized, but the planner is not completely\nfilesystem-free. For unit/repository verdicts it stats and hashes the shared\ninventory's members and resolves guard files such as ancestor markers, lock\nfiles, absolute file arguments, and ",(0,s.jsx)(t.code,{children:"invalidateOn"})," paths. A planner constructed\noutside the normal runner can also fall back to its own discovery walk when no\ninventory was seeded."]})]})}function h(e={}){let{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(a,{...e})}):a(e)}},98988(e,t,r){r.d(t,{R:()=>o,x:()=>c});var n=r(32537);let s={},i=n.createContext(s);function o(e){let t=n.useContext(i);return n.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function c(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),n.createElement(i.Provider,{value:t},e.children)}}}]);