1(self.webpackChunkdrill4j_github_io=self.webpackChunkdrill4j_github_io||[]).push([[4498],{3905:function(e,t,a){"use strict";a.d(t,{Zo:function(){return u},kt:function(){return d}});var n=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),p=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},u=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),m=p(a),d=r,k=m["".concat(s,".").concat(d)]||m[d]||c[d]||i;return a?n.createElement(k,l(l({ref:t},u),{},{components:a})):n.createElement(k,l({ref:t},u))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,l=new Array(i);l[0]=m;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o.mdxType="string"==typeof e?e:r,l[1]=o;for(var p=2;p<i;p++)l[p]=a[p];return n.createElement.apply(null,l)}return n.createElement.apply(null,a)}m.displayName="MDXCreateElement"},3756:function(e,t,a){"use strict";a.r(t),a.d(t,{frontMatter:function(){return l},metadata:function(){return o},toc:function(){return s},default:function(){return u}});var n=a(2122),r=a(9756),i=(a(7294),a(3905)),l={id:"etl-update-metrics",title:"Metrics update process"},o={unversionedId:"etl-update-metrics",id:"version-0.10.0/etl-update-metrics",isDocsHomePage:!1,title:"Metrics Update Process",description:"Overview",source:"@site/versioned_docs/version-0.10.0/etl-update-metrics.mdx",sourceDirName:".",slug:"/etl-update-metrics",permalink:"/docs/0.10.0/etl-update-metrics",version:"0.10.0",frontMatter:{id:"etl-update-metrics",title:"Metrics update process"},sidebar:"version-0.10.0/docs",previous:{title:"Drill4J Backend",permalink:"/docs/0.10.0/drill-services/drill-backend"},next:{title:"Drill4J UI",permalink:"/docs/0.10.0/drill-services/drill-ui"}},s=[{value:"Overview",id:"overview",children:[]},{value:"Scheduled Run",id:"scheduled-run",children:[]},{value:"On-Demand Run",id:"on-demand-run",children:[{value:"Incremental Updates",id:"incremental-updates",children:[]},{value:"Complete Restart",id:"complete-restart",children:[]}]},{value:"Data Retention and Cleanup",id:"data-retention-and-cleanup",children:[{value:"Configuring Retention Periods",id:"configuring-retention-periods",children:[]},{value:"Configuring Cleanup Schedule",id:"configuring-cleanup-schedule",children:[]}]},{value:"Fine-Tuning Performance",id:"fine-tuning-performance",children:[{value:"Extraction Limit",id:"extraction-limit",children:[]},{value:"Fetch Size",id:"fetch-size",children:[]},{value:"Buffer Size",id:"buffer-size",children:[]},{value:"Transformation Buffer Size",id:"transformation-buffer-size",children:[]},{value:"Batch Size",id:"batch-size",children:[]},{value:"Consistency Window",id:"consistency-window",children:[]},{value:"Processing Delay",id:"processing-delay",children:[]}]},{value:"Tracking and Monitoring",id:"tracking-and-monitoring",children:[{value:"Logging Levels",id:"logging-levels",children:[]},{value:"Tracking Progress",id:"tracking-progress",children:[]}]},{value:"Troubleshooting",id:"troubleshooting",children:[{value:"ETL Pipeline Fails",id:"etl-pipeline-fails",children:[]},{value:"ETL Running Slowly",id:"etl-running-slowly",children:[]},{value:"Metrics Data Inconsistency",id:"metrics-data-inconsistency",children:[]}]}],p={toc:s};function u(e){var t=e.components,a=(0,r.Z)(e,["components"]);return(0,i.kt)("wrapper",(0,n.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h2",{id:"overview"},"Overview"),(0,i.kt)("p",null,"Drill4J uses an ETL (Extract, Transform, Load) process to transform raw data collected by agents into actionable metrics.\nThe system maintains two separate database schemas:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"raw_data")," - Stores data sent by agents in its original format without any processing."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"metrics")," - Contains processed data that powers the dashboards and API responses. This schema is populated and maintained by the ETL pipeline.")),(0,i.kt)("p",null,"The ETL process runs automatically on a schedule and can also be triggered manually.\nIt reads from ",(0,i.kt)("inlineCode",{parentName:"p"},"raw_data"),", performs necessary transformations and calculations, and updates the ",(0,i.kt)("inlineCode",{parentName:"p"},"metrics")," schema.\nThis architecture allows for data reprocessing if needed and separates data collection from data analysis concerns."),(0,i.kt)("h2",{id:"scheduled-run"},"Scheduled Run"),(0,i.kt)("p",null,"ETL process runs automatically using a Cron job.\nThe schedule is controlled by the ",(0,i.kt)("inlineCode",{parentName:"p"},"DRILL_SCHEDULER_ETL_JOB_CRON")," environment variable passed to the Drill4J Backend."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Note:")," Applying changes to the environment variable requires restarting the Drill4J Backend instance for the new schedule to take effect."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Best Practice:")," Adjust the ETL schedule frequency so that job execution finishes before the next run time. Although the job won't start a new run until the previous one finishes, we recommend leaving extra buffer time."),(0,i.kt)("h2",{id:"on-demand-run"},"On-Demand Run"),(0,i.kt)("p",null,"You can manually trigger the ETL process without waiting for the scheduled execution.\nThe ETL process supports two modes of operation:"),(0,i.kt)("h3",{id:"incremental-updates"},"Incremental Updates"),(0,i.kt)("p",null,"By default, the ETL process performs incremental updates, processing only new data from ",(0,i.kt)("inlineCode",{parentName:"p"},"raw_data")," that hasn't been transformed yet. This is the most efficient approach for day-to-day operations."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"When to use:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Regular scheduled updates"),(0,i.kt)("li",{parentName:"ul"}
1,"Quick catch-up after a brief period"),(0,i.kt)("li",{parentName:"ul"},"Before querying impacted tests/methods API endpoints")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"API Request:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"curl -X 'POST' \\\n 'http://localhost:8090/api/metrics/refresh?reset=false' \\\n -H 'accept: application/json' \\\n -H 'X-API-KEY: [YOUR_API_KEY]' \\\n -d ''\n")),(0,i.kt)("h3",{id:"complete-restart"},"Complete Restart"),(0,i.kt)("p",null,"A complete restart clears all data from the ",(0,i.kt)("inlineCode",{parentName:"p"},"metrics")," schema and reprocesses everything from scratch based on available data in ",(0,i.kt)("inlineCode",{parentName:"p"},"raw_data"),"."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"When to use:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"When the settings affecting the metrics changed (metrics period expanded, rules for ignoring methods and classes changed)"),(0,i.kt)("li",{parentName:"ul"},"When retrospective changes have occurred"),(0,i.kt)("li",{parentName:"ul"},"When data integrity issues are suspected"),(0,i.kt)("li",{parentName:"ul"},"After schema migrations or updates")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"API Request:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"curl -X 'POST' \\\n 'http://localhost:8090/api/metrics/refresh?reset=true' \\\n -H 'accept: application/json' \\\n -H 'X-API-KEY: [YOUR_API_KEY]' \\\n -d ''\n")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Warning:")," This operation is resource-intensive and can take a long time to complete (from hours to days, depending on the amount of data)."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Best Practice:")," Schedule complete restarts during maintenance windows when metrics access is not critical."),(0,i.kt)("h2",{id:"data-retention-and-cleanup"},"Data Retention and Cleanup"),(0,i.kt)("p",null,"Drill4J automatically manages data retention for both ",(0,i.kt)("inlineCode",{parentName:"p"},"raw_data")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"metrics")," schemas using dedicated cleanup jobs.\nThis prevents unlimited data growth and maintains optimal database performance."),(0,i.kt)("h3",{id:"configuring-retention-periods"},"Configuring Retention Periods"),(0,i.kt)("p",null,"Each agent group has its own retention settings that control how long data is kept in each schema."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Viewing Current Settings:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"curl -X 'GET' \\\n 'http://localhost:8090/api/group-settings/[GROUP_ID]' \\\n -H 'accept: application/json' \\\n -H 'X-API-KEY: [YOUR_API_KEY]'\n")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Example Response:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-json"},'{\n "retentionPeriodDays": 30, // cleanup window for raw_data schema\n "metricsPeriodDays": 30 // cleanup window for metrics schema\n}\n')),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Updating Retention Settings:")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Warning"),": Please note the body ",(0,i.kt)("strong",{parentName:"p"},"must contain all parameters")," with appropriate values - payload is not merged - it overwrites all settings."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"curl -X 'PUT' \\\n 'http://localhost:8090/api/group-settings/[GROUP_ID]' \\\n -H 'accept: application/json' \\\n -H 'X-API-KEY: [YOUR_API_KEY]' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"retentionPeriodDays\": 45,\n \"metricsPeriodDays\": 45\n }'\n")),(0,i.kt)("h3",{id:"configuring-cleanup-schedule"},"Configuring Cleanup Schedule"),(0,i.kt)("p",null,"The cleanup jobs run on a schedule controlled by the ",(0,i.kt)("inlineCode",{parentName:"p"},"DRILL_SCHEDULER_DATA_RETENTION_JOB_CRON")," environment variable."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Example Configuration:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yml"},"# Runs cleanup jobs once a day at midnight\nDRILL_SCHEDULER_DATA_RETENTION_JOB_CRON=0 0 0 * * ?\n")),(0,i.kt)("p",null,"By default, the cleanup job is set to run daily at 01:00."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Note:")," Changes to the environment variable require restarting the Drill4J Backend instance."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Best Practice:"),"\nThe ",(0,i.kt)("inlineCode",{parentName:"p"},"retentionPeriodDays")," for ",(0,i.kt)("inlineCode",{parentName:"p"},"raw_data")," should be greater than or equal to the ",(0,i.kt)("inlineCode",{parentName:"p"},"metricsPeriodDays"),". This ensures that all data required for a complete ETL reprocessing is available, allowing safe full metric recalculation if needed."),(0,i.kt)("h2",{id:"fine-tuning-performance"},"Fine-Tuning Performance"),(0,i.kt)("p",null,"The ETL pipeline can be tuned for optimal performance based on your infrastru
1cture and data volume. These parameters control memory usage, database interaction, and throughput."),(0,i.kt)("h3",{id:"extraction-limit"},"Extraction Limit"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_EXTRACTION_LIMIT")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Controls page size for extraction queries"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Adds a ",(0,i.kt)("inlineCode",{parentName:"li"},"LIMIT")," to the SQL extraction query used for each page. The extractor will keep requesting the next pages until there is no more data to extract"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Query latency and memory/CPU load per extraction request"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 1000000"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Decrease the limit if single extraction queries are slow"),(0,i.kt)("li",{parentName:"ul"},"Increase the limit if ETL is spending too much time paging and the database can handle larger result sets")))),(0,i.kt)("h3",{id:"fetch-size"},"Fetch Size"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_FETCH_SIZE")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," JDBC fetch size hint for SQL queries used by the data extractor"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Determines how many rows are fetched from the database per round trip"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Network latency and database load"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 2000"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase for better throughput on fast networks (5000-10000)"),(0,i.kt)("li",{parentName:"ul"},"Decrease for slower networks or smaller result sets (500-1000)")))),(0,i.kt)("h3",{id:"buffer-size"},"Buffer Size"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_BUFFER_SIZE")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Size of the in-memory buffer between data extractor and loaders"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Prevents unbounded memory growth. When the buffer is full, the extractor suspends, giving loaders time to process"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Affects throughput and memory usage"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 2000"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase for faster processing if memory allows (5000-20000)"),(0,i.kt)("li",{parentName:"ul"},"Decrease if experiencing memory pressure (500-1000)")))),(0,i.kt)("h3",{id:"transformation-buffer-size"},"Transformation Buffer Size"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_TRANSFORMATION_BUFFER_SIZE")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Controls how many aggregated rows the transformer accumulates in memory to pass aggregated results to loaders."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," The transformer groups and aggregates rows until this threshold is reached, then emits aggregated items downstream."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Larger values can improve throughput when aggregation significantly reduces cardinality, because loaders write fewer items"),(0,i.kt)("li",{parentName:"ul"},"Too-large values may increase heap usage and GC overhead and can lead to OOM on large/high-cardinality datasets"),(0,i.kt)("li",{parentName:"ul"},"Too-small values reduce aggregation opportunities and can increase the number of items written, slowing down loading"))),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 2000"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase (e.g., 4000\u201320000) if you have enough memory"),(0,i.kt)("li",{parentName:"ul"},"Decrease (e.g., 500\u20131000) if you observe memory pressure"),(0,i.kt)("li",{parentName:"ul"},"If increasing the buffer doesn\u2019t reduce load volume, you\u2019re likely dealing with high-cardinality ke
1ys (too many unique methods/tests).")))),(0,i.kt)("h3",{id:"batch-size"},"Batch Size"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_BATCH_SIZE")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Number of items grouped into a single write batch/transaction used by data loaders"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Controls commit frequency and transaction size"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Write performance and transaction overhead"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 1000"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase for better write performance (2000-5000)"),(0,i.kt)("li",{parentName:"ul"},"Decrease to reduce transaction lock time (100-500)")))),(0,i.kt)("h3",{id:"consistency-window"},"Consistency Window"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_CONSISTENCY_WINDOW")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Number of seconds to subtract from the last processed timestamp when starting an ETL run"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Allows re-processing records that arrived in the source database with a delay or were inconsistent at the time of the previous ETL run"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Slightly increases the amount of data reprocessed per run, but improves data consistency"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 0"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase this value if records are skipped because agents may send data out of order.")))),(0,i.kt)("h3",{id:"processing-delay"},"Processing Delay"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Environment Variable:")," ",(0,i.kt)("inlineCode",{parentName:"li"},"DRILL_ETL_PROCESSING_DELAY")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Purpose:")," Number of seconds to subtract from the current time when calculating the upper bound of the ETL processing window"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Behavior:")," Delays the ETL process to allow in-flight transactions in the data source to complete before they are processed"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Impact:")," Introduces a small lag in metrics freshness, but prevents partial or uncommitted data from being picked up"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Default:")," 0"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"Tuning Guidance:"),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Increase if ETL occasionally picks up incomplete or inconsistent records from active transactions (e.g., 5\u201360 seconds)")))),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Note:")," Applying changes to the environment variable requires restarting the Drill4J Backend instance for the new schedule to take effect."),(0,i.kt)("h2",{id:"tracking-and-monitoring"},"Tracking and Monitoring"),(0,i.kt)("p",null,"The ETL process provides comprehensive logging to help you monitor execution, troubleshoot issues, and optimize performance."),(0,i.kt)("h3",{id:"logging-levels"},"Logging Levels"),(0,i.kt)("p",null,"The ETL logging supports multiple levels:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"INFO:")," Logs only ETL start and completion events."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"DEBUG:")," In addition to INFO, logs when each extractor and loader starts and finishes."),
1(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("strong",{parentName:"li"},"TRACE:")," In addition to DEBUG, logs every batch commit during loading.")),(0,i.kt)("h3",{id:"tracking-progress"},"Tracking Progress"),(0,i.kt)("p",null,"You can track the real-time progress of ETL executions by calling the Metrics Refresh Status API:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"curl -X 'GET' \\\n 'http://localhost:8090/api/metrics/refresh-status' \\\n -H 'accept: application/json' \\\n -H 'X-API-KEY: [YOUR_API_KEY]'\n")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Example Response:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-json"},'"data": {\n "status": "LOADING", // Current ETL status\n "lastProcessedAt": "2025-11-01T09:59:30.000Z", // Timestamp of the last processed data item\n "lastRunAt": "2025-11-01T10:00:00.000Z", // Timestamp when the last run started\n "lastDuration": 1220, // Duration of the last ETL run in milliseconds\n "lastRowsProcessed": 18500 // Number of rows processed in the last ETL run\n}\n')),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"ETL Statuses:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"EXTRACTING"),": ETL process is extracting data from ",(0,i.kt)("inlineCode",{parentName:"li"},"raw_data")," schema."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"LOADING"),": ETL is actively loading data."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"SUCCESS"),": ETL completed successfully."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"FAILED"),": ETL run ended with an error. Check the ",(0,i.kt)("strong",{parentName:"li"},"Drill4J Admin")," container logs for details. It will try again on the next scheduled run, but it will continue to fail until the underlying issue is fixed.")),(0,i.kt)("h2",{id:"troubleshooting"},"Troubleshooting"),(0,i.kt)("h3",{id:"etl-pipeline-fails"},"ETL Pipeline Fails"),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Symptoms:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"ETL log shows errors"),(0,i.kt)("li",{parentName:"ul"},"Manual refresh API returns errors"),(0,i.kt)("li",{parentName:"ul"},"Metrics not updating")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Solutions:")),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Check Database Connectivity:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Verify database connection credentials"),(0,i.kt)("li",{parentName:"ul"},"Test database accessibility from the Backend instance"),(0,i.kt)("li",{parentName:"ul"},"Check firewall rules and network connectivity"))),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Verify Schema Existence:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Check database user has necessary permissions (SELECT, INSERT, UPDATE, DELETE)")))),(0,i.kt)("h3",{id:"etl-running-slowly"},"ETL Running Slowly"),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Symptoms:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"ETL process execution time keeps increasing"),(0,i.kt)("li",{parentName:"ul"},"Data processing delay continues to grow"),(0,i.kt)("li",{parentName:"ul"},"Metrics stop reflecting the most recent data")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Solutions:")),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Data Volume:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Review ",(0,i.kt)("a",{parentName:"li",href:"#configuring-retention-periods"},"retention settings")," - older data may not be needed"),(0,i.kt)("li",{parentName:"ul"},"Review methods ignore rules - consider excluding unneeded classes and methods"))),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Database Performance:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Consider increasing database resources (CPU, memory, IOPS)"),(0,i.kt)("li",{parentName:"ul"},"Consider database maintenance (VACUUM, ANALYZE, etc.)"))),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Review Performance Parameters:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Consider increasing ",(0,i.kt)("a",{parentName:"li",href:"#buffer-size"},(0,i.kt)("inlineCode",{parentName:"a"},"bufferSize")),", ",(0,i.kt)("a",{parentName:"li",href:"#fetch-size"},(0,i.kt)("inlineCode",{parentName:"a"},"fetchSize")),", or ",(0,i.kt)("a",{parentName:"li",href:"#batch-size"},(0,i.kt)("inlineCode",{parentName:"a"},"batchSize")),"."),(0,i.kt)("li",{parentName:"ul"},"Monitor memory usage when adjusting parameters")))),(0,i.kt)("h3",{id:"metrics-data-inconsistency"},"Metrics Data Inconsistency"),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Symptoms:")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Dashboard showing unexpected values"),(0,i.kt)("li",{parentName:"ul"},"API results don't match raw data")),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Solutions:")),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Perform Complete Refresh:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Use ",(0,i.kt)("a",{parentName:"li",href:"./etl-update-metrics#complete-restart"},(0,i.kt)("inlineCode",{parentName:"a"},"reset=true"))," API call to reprocess all data"))),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Check Data Retention:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Review ",(0,i.kt)("a",{parentName:"li",href:"#configuring-retention-periods"},(0,i.kt)("inlineCode",{parentName:"a"},"retentionPeriodDays"))," and ",(0,i.kt)("a",{parentName:"li",href:"#configuring-retention-periods"},(0,i.kt)("inlineCode",{parentName:"a"},"metricsPeriodDays"))," settings"),(0,i.kt)("li",{parentName:"ul"},"Verify cleanup jobs haven't removed needed data"))),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("p",{parentName:"li"},(0,i.kt)("strong",{parentName:"p"},"Investigate Errors:")),(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Review ETL logs for failure counts"),(0,i.kt)("li",{parentName:"ul"},"Review ETL Metadata table for error messages")))))}u.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.