1"use strict";(self.webpackChunkholistics_docs=self.webpackChunkholistics_docs||[]).push([["133"],{33798(e,n,t){t.r(n),t.d(n,{metadata:()=>i,default:()=>h,frontMatter:()=>a,contentTitle:()=>r,toc:()=>c,assets:()=>l});var i=JSON.parse('{"id":"docs/datahub-integration/setup","title":"Setup DataHub integration","description":"Step-by-step guide to set up Holistics metadata ingestion into DataHub","source":"@site/docs/docs/datahub-integration/setup.md","sourceDirName":"docs/datahub-integration","slug":"/docs/datahub-integration/setup","permalink":"/docs/datahub-integration/setup","draft":false,"unlisted":false,"tags":[],"version":"current","frontMatter":{"title":"Setup DataHub integration","slug":"/docs/datahub-integration/setup","description":"Step-by-step guide to set up Holistics metadata ingestion into DataHub"},"sidebar":"docs","previous":{"title":"DataHub integration","permalink":"/docs/datahub-integration/"},"next":{"title":"Local Agentic Dev","permalink":"/docs/development/local-agentic-development"}}'),s=t(74848),o=t(28453);let a={title:"Setup DataHub integration",slug:"/docs/datahub-integration/setup",description:"Step-by-step guide to set up Holistics metadata ingestion into DataHub"},r,l={},c=[{value:"Prerequisites",id:"prerequisites",level:2},{value:"Installation",id:"installation",level:2},{value:"Configuration",id:"configuration",level:2},{value:"Basic setup with local directory",id:"basic-setup-with-local-directory",level:3},{value:"Git-based setup",id:"git-based-setup",level:3},{value:"Connection mapping",id:"connection-mapping",level:3},{value:"Feature flags",id:"feature-flags",level:3},{value:"Filtering",id:"filtering",level:3},{value:"Stateful ingestion",id:"stateful-ingestion",level:3},{value:"Running the ingestion",id:"running-the-ingestion",level:2},{value:"Verification",id:"verification",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2}];function d(e){let n={a:"a",admonition:"admonition",code:"code",h2:"h2",h3:"h3",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,o.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.admonition,{title:"Early access",type:"warning",children:(0,s.jsxs)(n.p,{children:["This feature is currently in development and not yet available. ",(0,s.jsx)(n.a,{href:"mailto:[email protected]",children:"Contact Holistics"})," to sign up for early access."]})}),"\n",(0,s.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,s.jsx)(n.p,{children:"Before setting up the integration, make sure you have:"}),"\n",(0,s.jsxs)(n.ul,{children:["\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Holistics CLI"})," installed and available in your PATH. See the ",(0,s.jsx)(n.a,{href:"/docs/cli/",children:"CLI documentation"})," for installation instructions."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"DataHub instance"})," running and accessible. This can be a local instance or DataHub Cloud."]}),"\n",(0,s.jsxs)(n.li,{children:[(0,s.jsx)(n.strong,{children:"Access to your Holistics AML project"})," - either as a local directory or a git repository."]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"installation",children:"Installation"}),"\n",(0,s.jsx)(n.p,{children:"Install the DataHub Holistics connector using pip:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"pip install datahub-holistics\n"})}),"\n",(0,s.jsxs)(n.p,{children:["This package registers itself as a DataHub ingestion source plugin, so you can use ",(0,s.jsx)(n.code,{children:"type: holistics"})," in your ingestion recipes."]}),"\n",(0,s.jsx)(n.p,{children:"The connector expects a recent Holistics CLI that exposes the canonical lineage graph via:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"holistics aml lineage .\n"})}),"\n",(0,s.jsx)(n.h2,{id:"configuration",children:"Configuration"}),"\n",(0,s.jsx)(n.p,{children:"The connector is configured through a YAML recipe file, following DataHub's standard ingestion format."}),"\n",(0,s.jsx)(n.h3,{id:"basic-setup-with-local-directory",children:"Basic setup with local directory"}),"\n",(0,s.jsxs)(n.p,{children:["If your AML project is on your local machine, use the ",(0,s.jsx)(n.code,{children:"base_folder"})," option:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"source:\n type: holistics\n config:\n base_folder: /path/to/your/holistics-aml-project\n\n connection_to_platform_map:\n bigquery_prod:\n platform: bigquery\n env: PROD\n\nsink:\n type: datahub-rest\n
1config:\n server: http://localhost:8080\n"})}),"\n",(0,s.jsx)(n.h3,{id:"git-based-setup",children:"Git-based setup"}),"\n",(0,s.jsx)(n.p,{children:"For production use, you'll typically want the connector to clone your AML project from git. This ensures you're always ingesting from the latest committed state:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"source:\n type: holistics\n config:\n git_info:\n repo: https://github.com/your-company/holistics-project\n branch: main\n deploy_key_file: /path/to/deploy_key # Optional, for private repos\n\n connection_to_platform_map:\n bigquery_prod:\n platform: bigquery\n env: PROD\n\nsink:\n type: datahub-rest\n config:\n server: http://localhost:8080\n"})}),"\n",(0,s.jsx)(n.h3,{id:"connection-mapping",children:"Connection mapping"}),"\n",(0,s.jsx)(n.p,{children:"Connection mapping is essential for establishing lineage from your Holistics models to the underlying database tables. Without it, the connector won't know which DataHub platform corresponds to each Holistics data source."}),"\n",(0,s.jsxs)(n.p,{children:["Each entry maps a Holistics ",(0,s.jsx)(n.code,{children:"data_source_name"})," (as defined in your AML files) to a DataHub platform:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"connection_to_platform_map:\n # Simple mapping - just specify the platform\n bigquery_prod:\n platform: bigquery\n env: PROD\n\n # Detailed mapping - useful for platforms that need more context\n postgres_analytics:\n platform: postgres\n platform_instance: analytics-db\n database: analytics\n schema: public\n env: PROD\n\n # Snowflake example\n snowflake_warehouse:\n platform: snowflake\n platform_instance: my-snowflake\n env: PROD\n"})}),"\n",(0,s.jsx)(n.p,{children:"The connector uses this mapping to construct proper DataHub URNs for source tables, enabling end-to-end lineage from dashboards down to database tables."}),"\n",(0,s.jsx)(n.h3,{id:"feature-flags",children:"Feature flags"}),"\n",(0,s.jsx)(n.p,{children:"You can control what metadata gets extracted:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"source:\n type: holistics\n config:\n base_folder: /path/to/project\n\n # Feature flags (all default to true)\n extract_owners: true # Extract owner information from AML\n extract_lineage: true # Build lineage relationships\n extract_descriptions: true # Include descriptions from AML\n include_hidden_fields: false # Include fields marked as hidden\n\n # Platform identification\n platform_instance: production-holistics\n env: PROD\n\n connection_to_platform_map:\n # ... your mappings\n"})}),"\n",(0,s.jsx)(n.h3,{id:"filtering",children:"Filtering"}),"\n",(0,s.jsx)(n.p,{children:"Use regex patterns to control which entities get ingested:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:'source:\n type: holistics\n config:\n base_folder: /path/to/project\n\n # Only ingest specific entities\n model_pattern:\n allow:\n - ".*"\n deny:\n - "tmp_.*" # Skip temporary models\n - "test_.*" # Skip test models\n\n dataset_pattern:\n allow:\n - ".*"\n\n dashboard_pattern:\n allow:\n - ".*"\n\n connection_to_platform_map:\n # ... your mappings\n'})}),"\n",(0,s.jsx)(n.h3,{id:"stateful-ingestion",children:"Stateful ingestion"}),"\n",(0,s.jsx)(n.p,{children:"Enable stateful ingestion to automatically detect and remove stale entities when they're deleted from your AML project:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"source:\n type: holistics\n config:\n base_folder: /path/to/project\n\n stateful_ingestion:\n enabled: true\n\n connection_to_platform_map:\n # ... your mappings\n"})}),"\n",(0,s.jsx)(n.h2,{id:"running-the-ingestion",children:"Running the ingestion"}),"\n",(0,s.jsxs)(n.p,{children:["Save your recipe to a file (e.g., ",(0,s.jsx)(n.code,{children:"holistics_recipe.yaml"}),") and run:"]}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"datahub ingest -c holistics_recipe.yaml\n"})}),"\n",(0,s.jsx)(n.p,{children:"The connector will output progress information showing how many models, datasets, dashboards, and charts were processed."}),"\n",(0,s.jsx)(n.p,{children:"Internally, the connector calls the Holistics CLI and reconstructs DataHub entities from AML-native graph nodes and edges."}),"\n",(0,s.jsx)(n.h2,{id:"verification",children:"Verification"}),"\n",(0,s.jsx)(n.p,{children:"After the ingestion completes:"}),"\n",(0,s.jsxs)(n.ol,{children:["\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Check DataHub UI"}),' - Navigate to your DataHub instance and search for "holistics". You should see your models, datasets, and dashboards.']}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Verify lineage"})," - Open a dashboard and check the Lineage tab. You should see connections to charts, which connect to models, which connect to source tables."]}),"\n",(0,s.jsx)(n.p,{children:"The canonical AML graph may contain additional concepts such as filter blocks or other non-viz dashboard blocks. These are preserved in the CLI output but are not currently emitted as DataHub chart entities."}),"\n"]}),"\n",(0,s.jsxs)(n.li,{children:["\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Check schema"})," - Open a model and look at the Schema tab. Dimensions and measures should appear as fields with appropriate tags."]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(n.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"CLI not found"}),": Ensure the Holistics CLI is installed and in your PATH. Test by running ",(0,s.jsx)(n.code,{children:"holistics --version"}),"."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Git clone fails"}),": For private repositories, make sure your deploy key has read access and the path in ",(0,s.jsx)(n.code,{children:"deploy_key_file"})," is correct."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"No lineage to source tables"}),": Verify your ",(0,s.jsx)(n.code,{children:"connection_to_platform_map"})," entries match the ",(0,s.jsx)(n.code,{children:"data_source_name"})," values in your AML models."]}),"\n",(0,s.jsxs)(n.p,{children:[(0,s.jsx)(n.strong,{children:"Entities missing"}),": Check the ingestion report for filtered or errored entities. Adjust your ",(0,s.jsx)(n.code,{children:"*_pattern"})," settings if needed."]})]})}function h(e={}){let{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(d,{...e})}):d(e)}},28453(e,n,t){t.d(n,{R:()=>a,x:()=>r});var i=t(96540);let s={},o=i.createContext(s);function a(e){let n=i.useContext(o);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:a(e.components),i.createElement(o.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.