1"use strict";(self.webpackChunkdataedo=self.webpackChunkdataedo||[]).push([["17068"],{80453:function(e,a,t){t.r(a),t.d(a,{frontMatter:()=>l,default:()=>g,toc:()=>c,metadata:()=>i,assets:()=>r,contentTitle:()=>d});var i=JSON.parse('{"id":"data-sources/databases/databricks/databricks-unity-catalog-data-lineage","title":"Databricks Unity Catalog - Automatic data lineage","description":"","source":"@site/docs/data-sources/databases/databricks/databricks-unity-catalog-data-lineage.mdx","sourceDirName":"data-sources/databases/databricks","slug":"/docs/documenting-technology/supported-databases/databricks-unity-catalog-data-lineage","permalink":"/docs/documenting-technology/supported-databases/databricks-unity-catalog-data-lineage","draft":false,"unlisted":false,"tags":[],"version":"current","lastUpdatedAt":1783594224000,"sidebarPosition":2,"frontMatter":{"title":"Databricks Unity Catalog - Automatic data lineage","description":"","slug":"/docs/documenting-technology/supported-databases/databricks-unity-catalog-data-lineage","sidebar_position":2},"sidebar":"tutorialSidebar","previous":{"title":"Connecting to Databricks Unity Catalog","permalink":"/docs/documenting-technology/supported-databases/connecting-to-databricks-unity-catalog"},"next":{"title":"Databricks support (with Hive Metastore)","permalink":"/docs/documenting-technology/supported-databases/databricks-hive-metastore"}}'),n=t(85893),s=t(50065),o=t(88046);let l={title:"Databricks Unity Catalog - Automatic data lineage",description:"",slug:"/docs/documenting-technology/supported-databases/databricks-unity-catalog-data-lineage",sidebar_position:2},d=void 0,r={},c=[{value:"What to expect",id:"what-to-expect",level:2},{value:"Within Databricks",id:"within-databricks",level:3},{value:"Delta Live Tables lineage",id:"delta-live-tables-lineage",level:4},{value:"Other objects lineage",id:"other-objects-lineage",level:4},{value:"Between Databricks and external sources",id:"between-databricks-and-external-sources",level:3},{value:"External Locations",id:"external-locations",level:4},{value:"Fivetran Data Sources",id:"fivetran-data-sources",level:4},{value:"Known Limitations",id:"known-limitations",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"I don't see data lineage",id:"i-dont-see-data-lineage",level:3},{value:"I don't see data lineage from External location storage",id:"i-dont-see-data-lineage-from-external-location-storage",level:3},{value:"I don't see data lineage between objects from different catalogs",id:"i-dont-see-data-lineage-between-objects-from-different-catalogs",level:3},{value:"Data lineage is inaccurate",id:"data-lineage-is-inaccurate",level:3}];function h(e){let a={a:"a",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",strong:"strong",ul:"ul",...(0,s.a)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(a.h2,{id:"what-to-expect",children:"What to expect"}),"\n",(0,n.jsx)(a.h3,{id:"within-databricks",children:"Within Databricks"}),"\n",(0,n.jsxs)(a.p,{children:["Dataedo uses a built-in Databricks data lineage module that stores historic information. Column-level lineage will be created for various kinds of ",(0,n.jsx)(a.strong,{children:"tables"})," and ",(0,n.jsx)(a.strong,{children:"views"})," that exist in Databricks Unity Catalog. If many catalogs are documented in Dataedo, lineage between them will also be imported. Lineage appearance will change depending on whether the object is created using Delta Live Table (DLT) pipeline or not:"]}),"\n",(0,n.jsx)(a.h4,{id:"delta-live-tables-lineage",children:"Delta Live Tables lineage"}),"\n",(0,n.jsx)(a.p,{children:"On the lineage graph, the user can see the DLT pipeline object responsible for creating objects and loading data."}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-data-lineage/25a2718ad87765494adc53da4adfb581.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsx)(a.p,{children:"It is also possible to hide pipelines and view lineage directly between objects:"}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-data-lineage/84ca4d2cfa2e1fb5a145c76c4b155567.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsx)(a.h4,{id:"other-objects-lineage",children:"Other objects lineage"}),"\n",(0,n.jsx)(a.p,{children:"If an object is created without a DLT pipeline, for example by running a notebook with Python code, executing CREATE.. AS SELECT.. or CREATE VIEW.. SQL statements, lineage will always be shown directly between objects:"}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-data-lineage/5e1fea33098abb33dd5b989c6e29bb1d.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsx)(a.h3,{id:"between-databricks-and-external-sources",children:"Between Databricks and external sources"}
1),"\n",(0,n.jsx)(a.h4,{id:"external-locations",children:"External Locations"}),"\n",(0,n.jsx)(a.p,{children:"Dataedo creates lineage based on location URI when the data source of the table is located in one of the following cloud storages:"}),"\n",(0,n.jsxs)(a.ul,{children:["\n",(0,n.jsx)(a.li,{children:"Google Cloud"}),"\n",(0,n.jsx)(a.li,{children:"Amazon S3"}),"\n",(0,n.jsx)(a.li,{children:"Azure"}),"\n",(0,n.jsx)(a.li,{children:"Cloudflare"}),"\n"]}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-data-lineage/814758f939769c0e60a0363232a72706.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsx)(a.h4,{id:"fivetran-data-sources",children:"Fivetran Data Sources"}),"\n",(0,n.jsxs)(a.p,{children:["If you connect to some data source (for example Salesforce) using a Fivetran connection, you can utilize Dataedo ",(0,n.jsx)(a.a,{href:"/docs/documenting-technology/supported-databases/fivetran",children:"Fivetran connector"})," to create column-level lineage between this source and Databricks. On the diagram, you can see the Fivetran ETL program that processes data, which can be hidden as in the case of DLT pipelines to show lineage between objects directly:"]}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-data-lineage/f487165537ecceb156e5b91d3237be68.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsx)(a.h2,{id:"known-limitations",children:"Known Limitations"}),"\n",(0,n.jsx)("div",{class:"info",children:(0,n.jsxs)("ul",{children:[(0,n.jsx)("li",{children:"Column-level lineage for external tables will be created only if the data source (for example JSON file) schema is automatically discovered by Databricks and column names are not changed."}),(0,n.jsx)("li",{children:"For Delta Live Tables (DLT), Dataedo will use data lineage only for the latest update and ignore any older updates."})]})}),"\n",(0,n.jsxs)(a.p,{children:["Because Dataedo uses the Databricks API, unless otherwise specified in the documentation, the following limitations regarding data lineage must be taken into account: ",(0,n.jsx)(a.a,{href:"https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/data-lineage#limitations",children:"Databricks lineage limitations"})]}),"\n",(0,n.jsx)(a.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,n.jsx)(a.h3,{id:"i-dont-see-data-lineage",children:"I don't see data lineage"}),"\n",(0,n.jsxs)(a.ol,{children:["\n",(0,n.jsx)(a.li,{children:"Check if the object that does not have lineage has it in Databricks. If not, lineage won't be created in Dataedo."}),"\n",(0,n.jsx)(a.li,{children:"Rerun the import of the source - maybe the schema was imported in an older version or the configuration was incorrect."}),"\n"]}),"\n",(0,n.jsx)(a.h3,{id:"i-dont-see-data-lineage-from-external-location-storage",children:"I don't see data lineage from External location storage"}),"\n",(0,n.jsxs)(a.ol,{children:["\n",(0,n.jsx)(a.li,{children:"Check if the object that does not have lineage has it in Databricks. If not, lineage won't be created in Dataedo."}),"\n",(0,n.jsx)(a.li,{children:"Make sure the object has Linked Source set, with correctly assigned storage."}),"\n"]}),"\n",(0,n.jsx)(o.Z,{src:"https://dataedo.com/asset/img/markdown/docs/databricks-unity-catalog-lineage/f8b2943c54f25e64a093673834951361.png#center",width:"60%",alt:"Image title"}),"\n",(0,n.jsxs)(a.ol,{start:"3",children:["\n",(0,n.jsx)(a.li,{children:"If you don't see Linked Source, check if you have permissions to the appropriate Databricks External Location."}),"\n",(0,n.jsx)(a.li,{children:"Rerun the import of the source - maybe the schema was imported in an older version or the configuration was incorrect."}),"\n"]}),"\n",(0,n.jsx)(a.h3,{id:"i-dont-see-data-lineage-between-objects-from-different-catalogs",children:"I don't see data lineage between objects from different catalogs"}),"\n",(0,n.jsxs)(a.ol,{children:["\n",(0,n.jsx)(a.li,{children:"Check if the object that does not have lineage has it in Databricks. If not, lineage won't be created in Dataedo."}),"\n",(0,n.jsx)(a.li,{children:"Make sure that you have both catalogs imported."}),"\n",(0,n.jsx)(a.li,{children:"Rerun the import of the catalog with outflow objects (those to which data should flow)."}),"\n"]}),"\n",(0,n.jsx)(a.h3,{id:"data-lineage-is-inaccurate",children:"Data lineage is inaccurate"}),"\n",(0,n.jsxs)(a.ol,{children:["\n",(0,n.jsx)(a.li,{children:"Make sure that there are recent updates available for pipelines."}),"\n"]})]})}function g(e={}){let{wrapper:a}={...(0,s.a)(),...e.components};return a?(0,n.jsx)(a,{...e,children:(0,n.jsx)(h,{...e})}):h(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.