https://ethpandaops.io/assets/js/c3157cf5.52e1f22c.js

1"use strict";(globalThis.webpackChunkpublic_docs=globalThis.webpackChunkpublic_docs||[]).push([[6498],{28453(e,t,n){n.d(t,{R:()=>o,x:()=>l});var a=n(96540);const s={},i=a.createContext(s);function o(e){const t=a.useContext(i);return a.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:o(e.components),a.createElement(i.Provider,{value:t},e.children)}},31394(e,t,n){n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>h,default:()=>p,frontMatter:()=>r,metadata:()=>a,toc:()=>d});var a=n(67004),s=n(74848),i=n(28453),o=n(66362),l=n(60966);const r={slug:"open-source-xatu-data",title:"Open Sourcing Xatu Data",description:"We're thrilled to share that the EthPandaOps Xatu dataset is now open source! The dataset contains a wealth of information about the Ethereum network, including detailed data on beacon chain events, mempool activity, and canonical chain events.",authors:["savid","samcm"],tags:["xatu","data","mainnet","sepolia","holesky"],image:"/img/blog/open-source-xatu-data.jpg",githubRepos:[{name:"xatu-data",url:"https://github.com/ethpandaops/xatu-data"}],relatedLinks:[{name:"Apache Parquet",url:"https://parquet.apache.org"},{name:"Clickhouse",url:"https://clickhouse.com/"}]},h=void 0,c={authorsImageUrls:[void 0,void 0]},d=[{value:"Introduction",id:"introduction",level:2},{value:"Why are we open sourcing the data?",id:"why-are-we-open-sourcing-the-data",level:2},{value:"What is Xatu?",id:"what-is-xatu",level:2},{value:"What's in the dataset?",id:"whats-in-the-dataset",level:2},{value:"Beacon API Events",id:"beacon-api-events",level:3},{value:"Mempool Events",id:"mempool-events",level:3},{value:"Canonical Events",id:"canonical-events",level:3},{value:"How do I use the data?",id:"how-do-i-use-the-data",level:2},{value:"Clickhouse",id:"clickhouse",level:3},{value:"Querying directly",id:"querying-directly",level:4},{value:"Inserting data",id:"inserting-data",level:4},{value:"Globbing",id:"globbing",level:4},{value:"Conclusion",id:"conclusion",level:2}];function u(e){const t={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,i.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(o.A,{children:(0,s.jsxs)(t.p,{children:["The Ethereum Foundation is running a Data Challenge for EIP4844! ",(0,s.jsx)(t.a,{href:"https://esp.ethereum.foundation/data-challenge-4844",children:"Click here"})," for more info."]})}),"\n",(0,s.jsx)(t.h2,{id:"introduction",children:"Introduction"}),"\n",(0,s.jsx)(t.p,{children:"We're thrilled to share that the EthPandaOps Xatu dataset is now open source!"}),"\n",(0,s.jsx)(t.p,{children:"The dataset contains a wealth of information about the Ethereum network, including detailed data on beacon chain events, mempool activity, and canonical chain events."}),"\n",(0,s.jsx)(t.p,{children:"Summary:"}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"The data is dedicated to the public domain under the CC BY 4.0 license"}),"\n",(0,s.jsxs)(t.li,{children:["The entire schema is available ",(0,s.jsx)(t.a,{href:"/data/xatu#schema",children:"here"})]}),"\n",(0,s.jsxs)(t.li,{children:["Data is partitioned by hour or day in ",(0,s.jsx)(t.a,{href:"https://parquet.apache.org",children:"Apache Parquet"})," files"]}),"\n",(0,s.jsxs)(t.li,{children:["We've already published the last 60 days of data for Mainnet, Holesky & Sepolia","\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"We'll publish everything we have for Mainnet over the following weeks -- it's just a lot of data!"}),"\n",(0,s.jsx)(t.li,{children:"Testnets will have the last 90 days of data published"}),"\n",(0,s.jsxs)(t.li,{children:["Check ",(0,s.jsx)(t.a,{href:"/data/xatu",children:"here"})," for detailed data availability"]}),"\n"]}),"\n"]}),"\n",(0,s.jsx)(t.li,{children:"We'll continue to update tables with new data, as well as add new tables"}),"\n"]}),"\n",(0,s.jsx)(t.h2,{id:"why-are-we-open-sourcing-the-data",children:"Why are we open sourcing the data?"}),"\n",(0,s.jsx)(t.p,{children:"We aim to empower researchers, developers, and enthusiasts to explore the Ethereum network in depth and contribute to its ongoing evolution. We believe that open access to high-quality data is essential for the success of the Ethereum ecosystem."}),"\n",(0,s.jsx)(t.p,{children:"By making this data publicly available we hope to foster greater understanding of the Ethereum network and drive advancements in areas such as protocol development, monitoring and performance optimization."}),"\n",(0,s.jsx)(t.h2,{id:"what-is-xatu",children:"What is Xatu?"}),"\n",(0,s.jsxs)(t.p,{children:[(0,s.jsx)(t.a,{href:"https://github.com/ethpandaops/xatu",children:"Xatu"}

1)," is a tool for collecting data from different components of the Ethereum network that we've been building for a while now. Since it's initial release in December 2022 we've been running Xatu internally to monitor the Ethereum network by storing data in ",(0,s.jsx)(t.a,{href:"https://clickhouse.com/",children:"Clickhouse"}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["In-house we're using it for monitoring, analysis and incident response. Notably it was used for the ",(0,s.jsx)(t.a,{href:"https://notes.ethereum.org/@samcm/big-blocks",children:"Big Blocks Test"})," on Goerli/Mainnet in 2023 to help decide the EIP4844 blob parameters. It has also been the first port of call for analysing how Dencun performed through the fork lifecycle of ",(0,s.jsx)(t.a,{href:"https://notes.ethereum.org/@ethpandaops/dencun-devnet-10-analysis",children:"Devnets"})," -> ",(0,s.jsx)(t.a,{href:"https://notes.ethereum.org/@ethpandaops/goerli-sf-0-analysis",children:"Testnets"})," -> ",(0,s.jsx)(t.a,{href:"../dencun-fork-analysis",children:"Mainnet"}),"."]}),"\n",(0,s.jsx)(t.h2,{id:"whats-in-the-dataset",children:"What's in the dataset?"}),"\n",(0,s.jsxs)(t.p,{children:["Xatu has it's fingers in a lot of pies so we categorize the data into a few different types. Check ",(0,s.jsx)(t.a,{href:"/data/xatu#schema",children:"here"})," for the complete schema and data availability date ranges."]}),"\n",(0,s.jsx)(t.h3,{id:"beacon-api-events",children:"Beacon API Events"}),"\n",(0,s.jsx)(o.A,{children:(0,s.jsx)(t.p,{children:"Last 60 days published"})}),"\n",(0,s.jsxs)(t.p,{children:["Events that are derived from the ",(0,s.jsx)(t.a,{href:"https://ethereum.github.io/beacon-APIs/#/Events/eventstream",children:"Beacon API Event Stream"})," via Xatu Sentry from all consensus clients in multiple regions and networks. All events are annotated with additional data to help with analysis. For example, the ",(0,s.jsx)(t.code,{children:"attestation"})," events have information about ",(0,s.jsx)(t.em,{children:"when"})," the attestation was seen, and even the ",(0,s.jsx)(t.code,{children:"validator_index"})," of the attestation. ",(0,s.jsx)(t.strong,{children:"Mainnet data exists from June 2023."})]}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:["650+ billion ",(0,s.jsx)(t.code,{children:"attestation"})," events","\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsx)(t.li,{children:"6TiB compressed, 300TiB uncompressed \ud83d\ude32"}),"\n"]}),"\n"]}),"\n",(0,s.jsxs)(t.li,{children:["50+ million ",(0,s.jsx)(t.code,{children:"block"})," events"]}),"\n",(0,s.jsxs)(t.li,{children:["50+ million ",(0,s.jsx)(t.code,{children:"blob_sidecar"})," events"]}),"\n",(0,s.jsx)(t.li,{children:"Plus more!"}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"mempool-events",children:"Mempool Events"}),"\n",(0,s.jsx)(o.A,{children:(0,s.jsxs)(t.p,{children:["Publishing soon ",(0,s.jsx)("sup",{children:"tm"})]})}),"\n",(0,s.jsxs)(t.p,{children:["Events that are derived from Xatu Mimicry which connects to the execution p2p network. We'll be publishing these events in the next few days. ",(0,s.jsx)(t.strong,{children:"Mainnet data exists from March 2023."})]}),"\n",(0,s.jsxs)(t.ul,{children:["\n",(0,s.jsxs)(t.li,{children:["3+ billion ",(0,s.jsx)(t.code,{children:"transaction"})," events"]}),"\n"]}),"\n",(0,s.jsx)(t.h3,{id:"canonical-events",children:"Canonical Events"}),"\n",(0,s.jsx)(o.A,{children:(0,s.jsxs)(t.p,{children:["Publishing soon ",(0,s.jsx)("sup",{children:"tm"})]})}),"\n",(0,s.jsxs)(t.p,{children:["We also derive events from the finalized chain which we call ",(0,s.jsx)(t.code,{children:"canonical"})," events. ",(0,s.jsx)(t.strong,{children:"Mainnet data exists from Beacon Chain genesis in December 2020."})]}),"\n",(0,s.jsxs)(t.p,{children:["These events are especially useful for analysis when compared to ",(0,s.jsx)(t.code,{children:"Beacon API Events"})," and ",(0,s.jsx)(t.code,{children:"Mempool Events"}),". For example, comparing when an attestation was seen on the network against when it was included in a beacon block, or comparing when a transaction was first seen in the mempool to when it was included in a block. We'll be publishing these events in the coming weeks."]}),"\n",(0,s.jsx)(t.h2,{id:"how-do-i-use-the-data",children:"How do I use the data?"}),"\n",(0,s.jsxs)(t.p,{children:["The data is stored in Apache Parquet files which are a columnar storage format that is highly optimized for analytics. You can read these files using a variety of tools including Python & Clickhouse. Check out the ",(0,s.jsx)(t.a,{href:"https://github.com/ethpandaops/xatu-data",children:"repo"})," for more information on how to get started."]}),"\n",(0,s.jsx)(t.h3,{id:"clickhouse",children:"Clickhouse"}),"\n",(0,s.jsxs)(t.p,{children:["Using Clickhouse is the simplest way to get started. You can use the ",(0,s.jsx)(t.code,{children:"clickhouse client"})," to query the data directly from the Parquet files. Check out the ",(0,s.jsx)(t.a,{href:"https://clickhouse.com/docs/en/install",children:"Clickhouse documentation"})," to get setup."]}),"\n",(0,s.jsx)(t.h4,{id:"querying-directly",children:"Querying directly"}),"\n",(0,s.jsx)(t.p,{children:"To query all attestations events in Sepolia on the 20th of March 2024 in the 13th hour you can use the following query:"}),"\n",(0,s.jsxs)(l.B_,{children:[(0,s.jsx)(l.BD,{children:"Query"}),(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-sql",children:"SELECT COUNT(*)\nFROM\n    url('https://data.ethpandaops.io/xatu/sepoli

1a/databases/default/beacon_api_eth_v1_events_attestation/2024/2/20/13.parquet', 'Parquet')\n"})})]}),"\n",(0,s.jsx)("img",{alt:"Querying Parquet directly from Clickhouse",class:"rounded",src:"/img/blog/example-1.jpg"}),"\n",(0,s.jsx)(t.h4,{id:"inserting-data",children:"Inserting data"}),"\n",(0,s.jsx)(t.p,{children:"You can also insert the data into a Clickhouse database to query it more easily. This is highly recommended for larger queries."}),"\n",(0,s.jsxs)(l.B_,{children:[(0,s.jsx)(l.BD,{children:"Query"}),(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-sql",children:"INSERT INTO default.beacon_api_eth_v1_events_attestation\nSELECT *\nFROM url('https://data.ethpandaops.io/xatu/sepolia/databases/default/beacon_api_eth_v1_events_attestation/2024/2/20/13.parquet', 'Parquet')\n"})})]}),"\n",(0,s.jsx)("img",{alt:"Inserting directly in to Clickhouse",class:"rounded",src:"/img/blog/example-3.jpg"}),"\n",(0,s.jsx)(t.h4,{id:"globbing",children:"Globbing"}),"\n",(0,s.jsx)(t.p,{children:"Clickhouse supports globbing so you can query multiple Parquet files at once. For example, to count the entire days worth of attestation events in Sepolia on the 20th of March 2024 you can use the following query:"}),"\n",(0,s.jsxs)(l.B_,{children:[(0,s.jsx)(l.BD,{children:"Query"}),(0,s.jsx)(t.pre,{children:(0,s.jsx)(t.code,{className:"language-sql",children:"SELECT COUNT(*)\nFROM\n    url('https://data.ethpandaops.io/xatu/sepolia/databases/default/beacon_api_eth_v1_events_attestation/2024/2/20/{0..23}.parquet', 'Parquet')\n"})})]}),"\n",(0,s.jsx)("img",{alt:"Querying Parquet directly from Clickhouse with globbing",class:"rounded",src:"/img/blog/example-2.jpg"}),"\n",(0,s.jsx)(t.h2,{id:"conclusion",children:"Conclusion"}),"\n",(0,s.jsxs)(t.p,{children:["We cannot wait to see what the community does with the data! If you cook something up please let us know! We'd love to hear about it. If you have any questions feel free to reach out to us on ",(0,s.jsx)(t.a,{href:"https://twitter.com/ethpandaops",children:"Twitter"}),". If you notice any issues please make an issue on the ",(0,s.jsx)(t.a,{href:"https://github.com/ethpandaops/xatu-data/issues",children:"repo"}),"."]}),"\n",(0,s.jsxs)(t.p,{children:["Happy querying and don't forget the ",(0,s.jsx)(t.a,{href:"https://esp.ethereum.foundation/data-challenge-4844",children:"Data Challenge"})," run by the Ethereum Foundation!"]}),"\n",(0,s.jsx)(t.p,{children:"Love,"}),"\n",(0,s.jsx)(t.p,{children:"EthPandaOps Team \u2764\ufe0f"})]})}function p(e={}){const{wrapper:t}={...(0,i.R)(),...e.components};return t?(0,s.jsx)(t,{...e,children:(0,s.jsx)(u,{...e})}):u(e)}},60966(e,t,n){n.d(t,{B_:()=>u,BD:()=>p});var a=n(96540);const s="details_bdAo",i="summary_dHHT",o="open_cntL",l="summaryIcon_sZ_e",r="summaryTitle_e1Hx",h="detailsContent_Jcqq";var c=n(34164),d=n(74848);function u({children:e,summary:t,open:n=!1}){const[u,p]=(0,a.useState)(n);let m=t,g=e;if(!m&&Array.isArray(e)){const t=e.find(e=>"Summary"===e?.type?.name||"string"==typeof e?.type&&"summary"===e?.type?.toLowerCase());t&&(m=t.props.children,g=e.filter(e=>"Summary"!==e?.type?.name&&("string"!=typeof e?.type||"summary"!==e?.type?.toLowerCase())))}return(0,d.jsxs)("div",{className:s,children:[(0,d.jsxs)("div",{className:(0,c.A)(i,u&&o),onClick:()=>{p(!u)},children:[(0,d.jsx)("span",{className:l,children:u?"\u25bc":"\u25ba"}),(0,d.jsx)("span",{className:r,children:m})]}),(0,d.jsx)("div",{className:(0,c.A)(h,u&&o),children:g})]})}function p({children:e}){return(0,d.jsx)(d.Fragment,{children:e})}},66362(e,t,n){n.d(t,{A:()=>o});n(96540);const a={alert:"alert_dmjp",alertIcon:"alertIcon_W31s",alertContent:"alertContent_sQ4d","alert--info":"alert--info_mi3q","alert--warning":"alert--warning_zHeT","alert--danger":"alert--danger_mlzQ","alert--success":"alert--success_m5Z0"};var s=n(34164),i=n(74848);function o({children:e,type:t="info"}){return(0,i.jsxs)("div",{className:(0,s.A)(a.alert,a[`alert--${t}`]),children:[(0,i.jsxs)("div",{className:a.alertIcon,children:["info"===t&&(0,i.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"

1currentColor",children:(0,i.jsx)("path",{d:"M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm0 15c-.55 0-1-.45-1-1v-4c0-.55.45-1 1-1s1 .45 1 1v4c0 .55-.45 1-1 1zm1-8h-2V7h2v2z"})}),"warning"===t&&(0,i.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor",children:(0,i.jsx)("path",{d:"M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-1 15h2v-2h-2v2zm0-4h2V7h-2v6z"})}),"danger"===t&&(0,i.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor",children:(0,i.jsx)("path",{d:"M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm0 11c-.55 0-1-.45-1-1V8c0-.55.45-1 1-1s1 .45 1 1v4c0 .55-.45 1-1 1zm1 4h-2v-2h2v2z"})}),"success"===t&&(0,i.jsx)("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor",children:(0,i.jsx)("path",{d:"M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-2 15l-5-5 1.41-1.41L10 14.17l7.59-7.59L19 8l-9 9z"})})]}),(0,i.jsx)("div",{className:a.alertContent,children:e})]})}},67004(e){e.exports=JSON.parse('{"permalink":"/posts/open-source-xatu-data","source":"@site/blog/2024-03-25-open-source-xatu-data/index.md","title":"Open Sourcing Xatu Data","description":"We\'re thrilled to share that the EthPandaOps Xatu dataset is now open source! The dataset contains a wealth of information about the Ethereum network, including detailed data on beacon chain events, mempool activity, and canonical chain events.","date":"2024-03-25T00:00:00.000Z","tags":[{"inline":true,"label":"xatu","permalink":"/posts/tags/xatu"},{"inline":true,"label":"data","permalink":"/posts/tags/data"},{"inline":true,"label":"mainnet","permalink":"/posts/tags/mainnet"},{"inline":true,"label":"sepolia","permalink":"/posts/tags/sepolia"},{"inline":true,"label":"holesky","permalink":"/posts/tags/holesky"}],"readingTime":5.15,"hasTruncateMarker":false,"authors":[{"name":"savid \ud83d\ude82","title":"DevOps Engineer","bio":"DevOps Engineer","url":"https://github.com/savid","github":"https://github.com/savid","twitter":"https://x.com/savid","imageURL":"/img/team/savid.jpeg","key":"savid","page":null},{"name":"samcm","title":"DevOps Engineer","bio":"DevOps Engineer","url":"https://github.com/samcm","github":"https://github.com/samcm","twitter":"https://x.com/samcmAU","imageURL":"/img/team/samcm.jpeg","key":"samcm","page":null}],"frontMatter":{"slug":"open-source-xatu-data","title":"Open Sourcing Xatu Data","description":"We\'re thrilled to share that the EthPandaOps Xatu dataset is now open source! The dataset contains a wealth of information about the Ethereum network, including detailed data on beacon chain events, mempool activity, and canonical chain events.","authors":["savid","samcm"],"tags":["xatu","data","mainnet","sepolia","holesky"],"image":"/img/blog/open-source-xatu-data.jpg","githubRepos":[{"name":"xatu-data","url":"https://github.com/ethpandaops/xatu-data"}],"relatedLinks":[{"name":"Apache Parquet","url":"https://parquet.apache.org"},{"name":"Clickhouse","url":"https://clickhouse.com/"}]},"unlisted":false,"prevItem":{"title":"Tracoor: Ethereum beacon data and execution trace explorer","permalink":"/posts/tracoor-debug-tool"},"nextItem":{"title":"Attacknet: Chaos engineering on Ethereum","permalink":"/posts/attacknet-introduction"}}')}}]);