1"use strict";(self.webpackChunkstarlake=self.webpackChunkstarlake||[]).push([[2837],{54508:(e,n,i)=>{i.r(n),i.d(n,{assets:()=>a,contentTitle:()=>l,default:()=>h,frontMatter:()=>d,metadata:()=>t,toc:()=>c});const t=JSON.parse('{"id":"capabilities/ingestion-xml","title":"XML Files Ingestion","description":"1. Row Tag Detection","source":"@site/docs/0400-capabilities/300-ingestion-xml.md","sourceDirName":"0400-capabilities","slug":"/capabilities/ingestion-xml","permalink":"/starflow/capabilities/ingestion-xml","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":300,"frontMatter":{},"sidebar":"starlakeSidebar","previous":{"title":"JSON Files Ingestion","permalink":"/starflow/capabilities/ingestion-json"},"next":{"title":"POSITION (Fixed-Width) Files Ingestion","permalink":"/starflow/capabilities/ingestion-position"}}');var r=i(74848),s=i(28453);const d={},l="XML Files Ingestion",a={},c=[{value:"1. Row Tag Detection",id:"1-row-tag-detection",level:2},{value:"2. Attribute Prefix Mapping",id:"2-attribute-prefix-mapping",level:2},{value:"3. Nested Structure Support",id:"3-nested-structure-support",level:2},{value:"4. XSD Validation",id:"4-xsd-validation",level:2},{value:"5. Encoding",id:"5-encoding",level:2},{value:"6. Row Filtering",id:"6-row-filtering",level:2},{value:"7. Type Validation",id:"7-type-validation",level:2},{value:"8. Required Field Check",id:"8-required-field-check",level:2},{value:"9. Column Renaming",id:"9-column-renaming",level:2},{value:"10. Privacy Transformations",id:"10-privacy-transformations",level:2},{value:"11. Computed Columns (Script)",id:"11-computed-columns-script",level:2},{value:"12. Ignored Columns",id:"12-ignored-columns",level:2},{value:"13. Default Values",id:"13-default-values",level:2},{value:"14. Foreign Keys",id:"14-foreign-keys",level:2},{value:"15. Pre/Post SQL Hooks",id:"15-prepost-sql-hooks",level:2},{value:"16. Rejection Routing",id:"16-rejection-routing",level:2},{value:"17. Post-Load Expectations",id:"17-post-load-expectations",level:2},{value:"18. Freshness Monitoring",id:"18-freshness-monitoring",level:2},{value:"19. Primary Key Declaration",id:"19-primary-key-declaration",level:2},{value:"20. Ingestion Metrics",id:"20-ingestion-metrics",level:2},{value:"Summary",id:"summary",level:2}];function o(e){const n={a:"a",code:"code",h1:"h1",h2:"h2",header:"header",hr:"hr",li:"li",p:"p",pre:"pre",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(n.header,{children:(0,r.jsx)(n.h1,{id:"xml-files-ingestion",children:"XML Files Ingestion"})}),"\n",(0,r.jsx)(n.h2,{id:"1-row-tag-detection",children:"1. Row Tag Detection"}),"\n",(0,r.jsxs)(n.p,{children:["Each XML file is parsed by identifying a repeating element that represents a single record. The ",(0,r.jsx)(n.code,{children:"rowTag"})," option specifies this element. It can be auto-detected or set explicitly."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n metadata:\n format: "XML"\n options:\n rowTag: "country"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"2-attribute-prefix-mapping",children:"2. Attribute Prefix Mapping"}),"\n",(0,r.jsxs)(n.p,{children:["XML node attributes are mapped to columns with a configurable prefix (default: ",(0,r.jsx)(n.code,{children:"_"}),") to avoid naming collisions with child elements."]}),"\n",(0,r.jsxs)(n.p,{children:["For example, ",(0,r.jsx)(n.code,{children:'<country name="France">'})," produces a column named ",(0,r.jsx)(n.code,{children:"_name"}),"."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n metadata:\n options:\n attributePrefix: "_"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"3-nested-structure-support",children:"3. Nested Structure Support"}),"\n",(0,r.jsxs)(n.p,{children:["Child elements are mapped as columns. Repeated child elements are treated as arrays with nested attributes, using ",(0,r.jsx)(n.code,{children:'type: "struct"'})," and ",(0,r.jsx)(n.code,{children:"array: true"}),"."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'attributes:\n - name: "_name"\n type: "string"\n - name: "rank"\n type: "int"\n - name: "neighbor"\n type: "struct"\n array: true\n attributes:\n - name: "_name"\n type: "string"\n - name: "_direction"\n type: "string"\n'})}),"\n",(0,r.jsx)(n.p,{children:"Warehouse support for nested types varies:"}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Warehouse"}),(0,r.jsx)(n.th,{children:"Nested Type"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"BigQuery"}),(0,r.jsx)(n.td,{children:"Native STRUCT and ARRAY"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Databricks"}),(0,r.jsx)(n.td,{children:"Native STRUCT and ARRAY"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Snowflake"}),(0,r.jsx)(n.td,{children:"VARIANT column"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Redshift"}),(0,r.jsx)(n.td,{children:"Requires flattening"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"DuckDB"}),(0,r.jsx)(n.td,{children:"Native STRUCT and ARRAY"})]})]})]}
1),"\n",(0,r.jsx)(n.h2,{id:"4-xsd-validation",children:"4. XSD Validation"}),"\n",(0,r.jsx)(n.p,{children:"An optional XSD schema can be provided for record-level validation. Records that do not conform to the XSD are rejected."}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n metadata:\n options:\n rowValidationXSDPath: "path/to/schema.xsd"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"5-encoding",children:"5. Encoding"}),"\n",(0,r.jsxs)(n.p,{children:["The file is read according to the configured encoding (default: ",(0,r.jsx)(n.code,{children:"UTF-8"}),"). Override with the ",(0,r.jsx)(n.code,{children:"encoding"})," property in the table metadata."]}),"\n",(0,r.jsx)(n.h2,{id:"6-row-filtering",children:"6. Row Filtering"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"filter"})," option at the table level applies a SQL WHERE clause to filter accepted data before writing. Applied after column renaming."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n filter: "rank <= 10"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"7-type-validation",children:"7. Type Validation"}),"\n",(0,r.jsxs)(n.p,{children:["Every record is validated against the type schema before loading. Each attribute is assigned a type backed by a regex pattern. Values that match are loaded into the target table; values that fail are rejected to the ",(0,r.jsx)(n.code,{children:"audit.rejected"})," table."]}),"\n",(0,r.jsxs)(n.p,{children:["Built-in types include: ",(0,r.jsx)(n.code,{children:"string"}),", ",(0,r.jsx)(n.code,{children:"int"}),", ",(0,r.jsx)(n.code,{children:"long"}),", ",(0,r.jsx)(n.code,{children:"double"}),", ",(0,r.jsx)(n.code,{children:"decimal"}),", ",(0,r.jsx)(n.code,{children:"boolean"}),", ",(0,r.jsx)(n.code,{children:"date"}),", ",(0,r.jsx)(n.code,{children:"timestamp"}),", and numerous ISO date/time variants (",(0,r.jsx)(n.code,{children:"ISO_DATE_TIME"}),", ",(0,r.jsx)(n.code,{children:"ISO_LOCAL_DATE"}),", ",(0,r.jsx)(n.code,{children:"RFC_1123_DATE_TIME"}),", etc.). Custom types with user-defined regex patterns are also supported."]}),"\n",(0,r.jsx)(n.h2,{id:"8-required-field-check",children:"8. Required Field Check"}),"\n",(0,r.jsxs)(n.p,{children:["Each attribute can be marked ",(0,r.jsx)(n.code,{children:"required: true"}),". When set, the field must be present and non-null in the source record or the row is rejected."]}),"\n",(0,r.jsx)(n.h2,{id:"9-column-renaming",children:"9. Column Renaming"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"rename"})," property maps a source element or attribute to a different target column name in the database. Table-level renaming is also supported via ",(0,r.jsx)(n.code,{children:"table.rename"}),"."]}),"\n",(0,r.jsx)(n.h2,{id:"10-privacy-transformations",children:"10. Privacy Transformations"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"privacy"})," property applies a transformation to sensitive column values during ingestion. Built-in algorithms:"]}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Algorithm"}),(0,r.jsx)(n.th,{children:"Description"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"NONE"})}),(0,r.jsx)(n.td,{children:"No transformation"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"HIDE"})}),(0,r.jsxs)(n.td,{children:["Replace with a fixed string (e.g., ",(0,r.jsx)(n.code,{children:"***"}),")"]})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"MD5"})}),(0,r.jsx)(n.td,{children:"MD5 hash"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"SHA1"})}),(0,r.jsx)(n.td,{children:"SHA-1 hash"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"SHA256"})}),(0,r.jsx)(n.td,{children:"SHA-256 hash"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"SHA512"})}),(0,r.jsx)(n.td,{children:"SHA-512 hash"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:(0,r.jsx)(n.code,{children:"AES"})}),(0,r.jsx)(n.td,{children:"AES encryption (requires parameters)"})]})]})]}),"\n",(0,r.jsxs)(n.p,{children:["SQL-based privacy expressions are also supported with the ",(0,r.jsx)(n.code,{children:"SQL:"})," prefix."]}),"\n",(0,r.jsx)(n.h2,{id:"11-computed-columns-script",children:"11. Computed Columns (Script)"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"script"})," property defines a Spark SQL expression to compute a derived column. Scripts can reference other columns, Spark SQL functions, and the file metadata column ",(0,r.jsx)(n.code,{children:"sl_input_file_name"}),". Scripted fields must appear at the end of the attribute list and are automatically made optional (nullable)."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'attributes:\n - name: "_name"\n type: "string"\n - name: "source_file"\n script: "regexp_extract(sl_input_file_name, \'.+/(.+)$\', 1)"\n - name: "loaded_at"\n script: "current_timestamp()"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"12-ignored-columns",children:"12. Ignored Columns"}),"\n",(0,r.jsxs)(n.p,{children:["Columns marked with ",(0,r.jsx)(n.code,{children:"ignore: true"})," are excluded from the target table but remain available for use in ",(0,r.jsx)(n.code,{children:"script"})," expressions."]}),"\n",(0,r.jsx)(n.h2,{id:"13-default-values",children:"13. Default Values"}),"\n",(0,r.jsxs)(n.p,{children:["Optional attributes can specify a ",(0,r.jsx)(n.code,{children:"default"})," value used when the source value is NULL. Not valid for struct or array types."]}),"\n",(0,r.jsx)(n.h2,{id:"14-foreign-keys",children:"14. Foreign Keys"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"foreignKey"})," property declares a relationship to another table. Supported syntaxes: ",(0,r.jsx)(n.code,{children:"table"}),", ",(0,r.jsx)(n.code,{children:"domain.table"}),", ",(0,r.jsx)(n.code,{children:"table.column"}),", ",(0,r.jsx)(n.code,{children:"domain.table.column"}),"."]}),"\n",(0,r.jsx)(n.h2,{id:"15-prepost-sql-hooks",children:"15. Pre/Post SQL Hooks"}),"\n",(0,r.jsxs)(n.p,{children:["SQL statements can be executed before and after ingestion via ",(0,r.jsx)(n.code,{children:"presql"})," and ",(0,r.jsx)(n.code,{children:"postsql"}),"."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n presql:\n - "DELETE FROM staging.countries WHERE batch_id = \'{{batch_id}}\'"\n postsql:\n - "ANALYZE TABLE countries COMPUTE STATISTICS"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"16-rejection-routing",children:"16. Rejection Routing"}),"\n",(0,r.jsxs)(n.p,{children:["Records that fail any validation check are not silently dropped \u2014 they are routed to the ",(0,r.jsx)(n.code,{children:"audit.rejected"})," table for inspection and reporting."]}),"\n",(0,r.jsx)(n.h2,{id:"17-post-load-expectations",children:"17. Post-Load Expectations"}),"\n",(0,r.jsxs)(n.p,{children:["After data is written to the target table, optional data quality assertions are evaluated. See the full ",(0,r.jsx)(n.a,{href:"/starflow/capabilities/expectations",children:"Expectations reference"})," for all 53 built-in macros covering completeness, validity, volume, schema, uniqueness, and numeric checks."]}),"\n",(0,r.jsx)(n.h2,{id:"18-freshness-monitoring",children:"18. Freshness Monitoring"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"freshness"})," property defines staleness thresholds for the loaded data."]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-yaml",children:'table:\n metadata:\n freshness:\n warn: "1d"\n error: "2d"\n'})}),"\n",(0,r.jsx)(n.h2,{id:"19-primary-key-declaration",children:"19. Primary Key Declaration"}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"primaryKey"})," property declares the table's primary key columns, used for merge operations and data quality."]}),"\n",(0,r.jsx)(n.h2,{id:"20-ingestion-metrics",children:"20. Ingestion Metrics"}),"\n",(0,r.jsxs)(n.p,{children:["Attributes can be tagged with a ",(0,r.jsx)(n.code,{children:"metricType"})," to compute statistics during ingestion:"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"CONTINUOUS"})," \u2014 min, max, mean, median, variance, standard deviation, skewness, kurtosis, percentiles."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"DISCRETE"})," \u2014 count distinct, category frequency, category count."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"TEXT"})," \u2014 text field statistics."]}),"\n"]}),"\n",(0,r.jsx)(n.hr,{}),"\n",(0,r.jsx)(n.h2,{id:"summary",children:"Summary"}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Check"}),(0,r.jsx)(n.th,{children:"Phase"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Row tag detection (",(0,r.jsx)(n.code,{children:"rowTag"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Attribute prefix mapping (",(0,r.jsx)(n.code,{children:"attributePrefix"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Nested structure parsing (",(0,r.jsx)(n.code,{children:"struct"}),", ",(0,r.jsx)(n.code,{children:"array"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["XSD validation (",(0,r.jsx)(n.code,{children:"rowValidationXSDPath"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Encoding"}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Pre-SQL hooks (",(0,r.jsx)(n.code,{children:"presql"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Type validation via regex"}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Required field presence"}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Column renaming (",(0,r.jsx)(n.code,{children:"rename"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Privacy transformations (",(0,r.jsx)(n.code,{children:"privacy"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Computed columns (",(0,r.jsx)(n.code,{children:"script"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Default values (",(0,r.jsx)(n.code,{children:"default"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Ignored columns (",(0,r.jsx)(n.code,{children:"ignore"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Row filtering (",(0,r.jsx)(n.code,{children:"filter"}),")"]}),(0,r.jsx)(n.td,{children:"Pre-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Rejection to ",(0,r.jsx)(n.code,{children:"audit.rejected"})]}),(0,r.jsx)(n.td,{children:"Load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsxs)(n.td,{children:["Post-SQL hooks (",(0,r.jsx)(n.code,{children:"postsql"}),")"]}),(0,r.jsx)(n.td,{children:"Post-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Post-load expectations"}),(0,r.jsx)(n.td,{children:"Post-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Freshness monitoring"}),(0,r.jsx)(n.td,{children:"Post-load"})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Ingestion metrics"}),(0,r.jsx)(n.td,{children:"Post-load"})]})]})]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(o,{...e})}):o(e)}},28453:(e,n,i)=>{i.d(n,{R:()=>d,x:()=>l});var t=i(96540);const r={},s=t.createContext(r);function d(e){const n=t.useContext(s);return t.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function l(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:d(e.components),t.createElement(s.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.