https://ethpandaops.io/assets/js/206c3ab2.4205a10b.js

1"use strict";(globalThis.webpackChunkpublic_docs=globalThis.webpackChunkpublic_docs||[]).push([[2532],{12586(e,t,n){n.r(t),n.d(t,{assets:()=>r,contentTitle:()=>l,default:()=>d,frontMatter:()=>o,metadata:()=>i,toc:()=>c});var i=n(72215),a=n(74848),s=n(28453);const o={slug:"the-lab",title:"Introducing The Lab",authors:["samcm","savid","matty"],description:"Introducing the new The Lab platform",tags:["data","lab"],image:"/img/blog/the-new-lab/header.png",relatedLinks:[{name:"The Lab",url:"https://lab.ethpandaops.io/"}]},l=void 0,r={authorsImageUrls:[void 0,void 0,void 0]},c=[{value:"Introduction",id:"introduction",level:2},{value:"The Lab",id:"the-lab",level:2},{value:"A fresh coat of paint",id:"a-fresh-coat-of-paint",level:3},{value:"Upcoming features",id:"upcoming-features",level:3},{value:"A transformational backend",id:"a-transformational-backend",level:2},{value:"The power of CBT <InlineLogo></InlineLogo>",id:"the-power-of-cbt-",level:3},{value:"New Clickhouse tables",id:"new-clickhouse-tables",level:4},{value:"Wrapping up",id:"wrapping-up",level:2}];function h(e){const t={a:"a",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",li:"li",ol:"ol",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components},{ClickableImage:n,Details:i,InlineLogo:o}=t;return n||p("ClickableImage",!0),i||p("Details",!0),o||p("InlineLogo",!0),(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(t.h2,{id:"introduction",children:"Introduction"}),"\n",(0,a.jsxs)(t.p,{children:["Almost a year ago, we quietly released an experimental version of The Lab with the goal of providing a public facing way to visualize the data we collect from the ",(0,a.jsx)(t.a,{href:"/data/xatu",children:"Xatu"})," ",(0,a.jsx)(o,{src:"/img/xatu.png",alt:"Xatu"})," project. We didn't really announce it anywhere, and were using it as a playground to experiment with new ideas. While it was a hit, it was limited in functionality and labor intensive to add new features to. As a result it has been sitting dormant for the last few months."]}),"\n",(0,a.jsxs)("center",{children:[(0,a.jsx)(t.p,{children:(0,a.jsx)(t.strong,{children:"Today we're excited to officially launch a brand new version of The Lab."})}),(0,a.jsxs)(t.p,{children:["Check it out here: ",(0,a.jsx)(t.a,{href:"https://lab.ethpandaops.io/",children:"lab.ethpandaops.io"})]})]}),"\n",(0,a.jsx)(t.h2,{id:"the-lab",children:"The Lab"}),"\n",(0,a.jsxs)(t.p,{children:["We set out to build a new version of The Lab with a single north star: ",(0,a.jsx)(t.strong,{children:"Make The Lab the community's first stop for verifying network performance."})," While we aren't quite there yet, we are aiming to rapidly iterate and improve the platform to get it to that point as soon as possible. Today's release aims to be at-parity with the previous experimental version of The Lab."]}),"\n",(0,a.jsx)(t.p,{children:"Since our team formed during The Merge, we've built numerous dashboards and tools to help us understand Ethereum network internals. This data requires deep context to interpret that our team already possesses. The Lab bridges this gap, allowing us to provide curated views of the data that are easy to understand."}),"\n",(0,a.jsxs)(t.p,{children:["Additionally, ",(0,a.jsx)(t.strong,{children:"all the raw data that you see in The Lab will be directly available to the community."})," Ethereum is built on the principles of openness and transparency, and we strongly believe in the power of open data. We are committed to continuing to build on these principles in the form of the ",(0,a.jsx)(t.a,{href:"/data/xatu",children:"Xatu"})," ",(0,a.jsx)(o,{src:"/img/xatu.png",alt:"Xatu"})," data project. For more info check out the ",(0,a.jsx)(t.a,{href:"#new-clickhouse-tables",children:"New Clickhouse Tables"})," section below."]}),"\n",(0,a.jsx)(t.p,{children:"Note: The Lab aims to provide a different perspective on the network that is complementary to existing explorers or monitoring tools that the community already uses, like Dora, Beaconcha.in, Etherscan, etc."}),"\n",(0,a.jsx)(t.h3,{id:"a-fresh-coat-of-paint",children:"A fresh coat of paint"}),"\n",(0,a.jsx)(t.p,{children:"We've completely redesigned The Lab with the polish and performance of a modern web application. This includes a new design system, 3 new themes (Light, Dark, and Star) and a refreshed approach to the user experience."}),"\n",(0,a.jsx)(n,{src:"/img/blog/the-new-lab/live.png",alt:"The new Lab",caption:"The Ethereum Live section within The Lab"}),"\n",(0,a.jsx)(t.h3,{id:"upcoming-features",children:"Upcoming features"}),"\n",(0,a.jsx)(t.p,{children:"We'll be adding a handful of new features to The Lab in the coming weeks. Some of these features are already in development but just need a little more time in the wok. Here's a sneak peek of what's coming:"}),"\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsxs)(t.li,{children:["New pages in the ",(0,a.jsx)(t.code,{children:"Consensus"})," section:","\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsxs)(t.li,{children:["A new ",(0,a.jsx)(t.code,{children:"Node"})," page which provides a detailed breakdown of the counts and versions of all the nodes on the network."]}),"\n",(0,a.jsxs)(t.li,{children:["A new ",(0,a.jsx)(t.code,{children:"Validator"})," page which provides a detailed view of a validator's performance over time, like when their attestation was seen by the Xatu nodes."]}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(t.li,{children:["A new ",(0,a.jsx)(t.code,{children:"Data Availability"})," section including analysis of:","\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsx)(t.li,{children:'A "Custody" deepdive which explores the data availability of our peers on the network (in collaboration with the EF P2P Team)'}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(t.li,{children:["A new ",(0,a.jsx)(t.code,{children:"Execution Layer"})," section including analysis of:","\n",(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsx)(t.li,{children:"State size"}),"\n",(0,a.jsx)(t.li,{children:"OPCODE and precompile usage"}),"\n",(0,a.jsxs)(t.li,{children:["Various state explorations, like size growth and hypothetical stateless explorations (in collaboration with the ",(0,a.jsx)(t.a,{href:"https://x.com/StatelessEth",children:"EF Stateless Consensus Team"}),")"]}),"\n",(0,a.jsxs)(t.li,{children:["Client execution time - how long it takes for all E

1xecution Layer clients to ",(0,a.jsx)(t.em,{children:"execute"})," a block"]}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(t.li,{children:["A new ",(0,a.jsx)(t.code,{children:"Fork Viewer"})," section which follows in the footsteps of ",(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/forky",children:"Forky"})," by providing a visual representation of the head of the chain ",(0,a.jsx)(t.em,{children:"of our peers"}),"."]}),"\n"]}),"\n",(0,a.jsx)(t.h2,{id:"a-transformational-backend",children:"A transformational backend"}),"\n",(0,a.jsxs)(t.p,{children:["We went back to the drawing board on a lot of the fundamental tooling in our existing data collection pipeline to help us achieve our goals. We ended up creating ",(0,a.jsx)(t.strong,{children:"8"})," new projects. Some of these projects are useful outside of the Ethereum ecosystem, and we're open sourcing them to the community."]}),"\n",(0,a.jsxs)(t.table,{children:[(0,a.jsx)(t.thead,{children:(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.th,{children:"Project"}),(0,a.jsx)(t.th,{children:"Description"})]})}),(0,a.jsxs)(t.tbody,{children:[(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/lab",children:"lab"})}),(0,a.jsx)(t.td,{children:"the new Lab frontend - built from the ground up."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/lab-backend",children:"lab-backend"})}),(0,a.jsx)(t.td,{children:"a new backend built from scratch to serve the Lab frontend."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/cbt",children:"clickhouse-build-tool (CBT)"})}),(0,a.jsx)(t.td,{children:"a simple, generic ClickHouse-focused data transformation tool that provides fast idempotent transformations with pure SQL or external scripts. Similar to DBT."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/cbt-api",children:"cbt-api"})}),(0,a.jsx)(t.td,{children:"an generic OpenAPI generator for CBT projects, providing both the spec & the server implementation to route queries to Clickhouse."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/xatu-cbt",children:"xatu-cbt"})}),(0,a.jsx)(t.td,{children:"contains the CBT models for the Xatu data."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/clickhouse-proto-gen",children:"clickhouse-proto-gen"})}),(0,a.jsx)(t.td,{children:"a tool for generating ClickHouse Protobuf schemas from Clickhouse tables."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/cartographoor",children:"cartographoor"})}),(0,a.jsx)(t.td,{children:"a single port-of-call for all our network specifications."})]}),(0,a.jsxs)(t.tr,{children:[(0,a.jsx)(t.td,{children:(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/xcli",children:"xcli"})}),(0,a.jsx)(t.td,{children:"a CLI for easily developing and testing new features in the Lab platform locally."})]})]})]}),"\n",(0,a.jsx)(t.p,{children:"These projects chain together to significantly simplify the process of building new features:"}),"\n",(0,a.jsxs)(t.ol,{children:["\n",(0,a.jsxs)(t.li,{children:["A new model is added to ",(0,a.jsx)(t.code,{children:"xatu-cbt"})]}),"\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"clickhouse-proto-gen"})," generates the Protobuf schemas for the new model."]}),"\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"cbt-api"})," generates the new OpenAPI spec, and the server implementation to route queries to Clickhouse."]}),"\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"lab"})," uses the new OpenAPI spec to generate typescript types for interacting with the new model within the Lab frontend."]}),"\n"]}),"\n",(0,a.jsxs)(t.p,{children:["The entire pipeline runs automatically through code generation, requiring no manual configuration. ","\ud83d\ude80"]}),"\n",(0,a.jsxs)(t.h3,{id:"the-power-of-cbt-",children:["The power of CBT ",(0,a.jsx)(o,{src:"/img/cbt.png",alt:"CBT"})]}),"\n",(0,a.jsxs)("div",{style:{display:"flex",flexWrap:"wrap",alignItems:"flex-start",ga

1p:"20px",marginTop:"20px"},children:[(0,a.jsxs)("div",{style:{flex:"1 1 300px",minWidth:"0"},children:[(0,a.jsxs)(t.p,{children:["The largest breakthrough in this new architecture is ",(0,a.jsx)(t.a,{href:"https://github.com/ethpandaops/cbt",children:"CBT"})," ",(0,a.jsx)(o,{src:"/img/cbt.png",alt:"CBT"}),". CBT lets us easily transform and compose data into interconnected tables, aggregating and consolidating it into new formats that build upon each other. The Lab is the most direct beneficiary of these new transformation tables, but the tables are also useful for other projects, both within ethPandaOps and the community."]}),(0,a.jsx)(t.p,{children:"This abstraction simplifies complexity while compounding the value of our data."})]}),(0,a.jsx)("img",{src:"/img/blog/the-new-lab/cbt-trojan-horse.png",alt:"CBT Trojan Horse",style:{maxWidth:"100%",height:"auto",width:"350px",flex:"0 0 auto"}})]}),"\n",(0,a.jsxs)(t.p,{children:[(0,a.jsx)(t.a,{href:"https://cbt.mainnet.ethpandaops.io%22",children:"CBT has a web interface"})," which provides a visual representation of the data transformation pipeline in real time. For example, the ",(0,a.jsx)(t.code,{children:"fct_block"})," table is built on top of other CBT",(0,a.jsx)(o,{src:"/img/cbt.png",alt:"CBT"})," tables in the form of ",(0,a.jsx)(t.code,{children:"int_block_canonical"})," and ",(0,a.jsx)(t.code,{children:"fct_block_head"}),". We'll be making a blog post about the inner workings of CBT soon."]}),"\n",(0,a.jsx)(n,{src:"/img/blog/the-new-lab/cbt-mainnet.png",alt:"CBT Mainnet showing the dependencies of the fct_block table",caption:"CBT Mainnet showing the dependencies of the fct_block table"}),"\n",(0,a.jsx)(t.h4,{id:"new-clickhouse-tables",children:"New Clickhouse tables"}),"\n",(0,a.jsxs)(t.p,{children:["We'll be publishing over 40 new transformation tables to the ",(0,a.jsx)(t.a,{href:"/data/xatu",children:"Xatu Dataset"})," ",(0,a.jsx)(o,{src:"/img/xatu.png",alt:"Xatu"})," in the coming weeks. These tables directly power The Lab, meaning ",(0,a.jsxs)(t.strong,{children:["everything you see in The Lab will be directly available to the community via the ",(0,a.jsx)(t.a,{href:"/data/xatu",children:"Xatu Dataset"})," ",(0,a.jsx)(o,{src:"/img/xatu.png",alt:"Xatu"}),"."]})," These tables are nearly always faster to query than the existing raw tables that we already publish, so if you want to build something new on top of the data we publish, these tables will be a great place to start."]}),"\n",(0,a.jsxs)(i,{children:[(0,a.jsx)("summary",{children:"View all new tables"}),(0,a.jsxs)(t.ul,{children:["\n",(0,a.jsx)(t.li,{children:"fct_attestation_correctness_by_validator_canonical"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_correctness_by_validator_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_correctness_canonical"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_correctness_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_first_seen_chunked_50ms"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_liveness_by_entity_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_attestation_observation_by_node"}),"\n",(0,a.jsx)(t.li,{children:"fct_block"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_blob_count"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_blob_count_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_blob_first_seen_by_node"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_data_column_sidecar_first_seen_by_node"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_first_seen_by_node"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_mev"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_mev_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_proposer"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_proposer_entity"}),"\n",(0,a.jsx)(t.li,{children:"fct_block_proposer_head"}),"\n",(0,a.jsx)(t.li,{children:"fct_data_column_availability_by_epoch"}),"\n",(0,a.jsx)(t.li,{children:"fct_data_column_availability_by_slot"}),"\n",(0,a.jsx)(t.li,{children:"fct_data_column_availability_by_slot_blob"}),"\n",(0,a.jsx)(t.li,{children:"fct_head_first_seen_by_node"}),"\n",(0,a.jsx)(t.li,{children:"fct_mev_bid_count_by_builder"}),"\n",(0,a.jsx)(t.li,{children:"fct_mev_bid_count_by_relay"}),"\n",(0,a.jsx)(t.li,{children:"fct_mev_bid_highest_value_by_builder_chunked_50ms"}),"\n",(0,a.jsx)(t.li,{children:"fct_prepared_block"}),"\n",(0,a.jsx)(t.li,{children:"int_attestation_attested_canonical"}),"\n",(0,a.jsx)(t.li,{children:"int_attestation_attested_head"}),"\n",(0,a.jsx)(t.li,{children:"int_attestation_first_seen"}),"\n",(0,a.jsx)(t.li,{children:"int_beacon_committee_head"}),"\n",(0,a.jsx)(t.li,{children:"int_block_blob_count_canonical"}),"\n",(0,a.jsx)(t.li,{children:"int_block_canonical"}),"\n",(0,a.jsx)(t.li,{children:"int_block_mev_canonical"}),"\n",(0,a.jsx)(t.li,{children:"int_address_first_access"}),"\n",(0,a.jsx)(t.li,{children:"int_address_last_access"}),"\n",(0,a.jsx)(t.li,{children:"int_address_storage_slot_first_access"}),"\n",(0,a.jsx)(t.li,{children:"int_address_storage_slot_last_access"}),"\n"]})]}),"\n",(0,a.jsx)(t.h2,{id:"wrapping-up",children:"Wrapping up"}),"\n",(0,a.jsxs)(t.p,{children:["We're extremely excited to be launching The Lab and we hope you enjoy using it. We'd love to hear your feedback or suggestions, please reach out to us on ",(0,a.jsx)(t.a,{href:"https://x.com/ethpandaops",children:"Twitter"})," or join the ",(0,a.jsx)(t.a,{href:"https://t.me/+JanoQFu_nO8yNzQ1",children:"Xatu Telegram Group"}),"."]}),"\n",(0,a.jsxs)("div",{style:{textAlign:"center",marginTop:"40px",fontSize:"1.5rem",fontWeight:"bold"},children:[(0,a.jsx)(t.p,{children:"See you soon for Fusaka!"}),(0,a.jsx)(t.p,{children:"\ud83d\ude80\ud83d\ude80\ud83d\ude80"})]})]})}function d(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,a.jsx)(t,{...e,children:(0,a.jsx)(h,{...e})}):h(e)}function p(e,t){throw new Error("Expected "+(t?"component":"object")+" `"+e+"` to be defined: you likely forgot to import, pass, or provide it.")}},28453(e,t,n){n.d(t,{R:()=>o,x:()=>l});var i=n(96540);const a={},s=i.createContext(a);function o(e){const t=i.useContext(s);return i.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:o(e.components),i.createElement(s.Provider,{value:t},e.children)}},72215(e){e.exports=JSON.parse('{"permalink":"/posts/the-lab","source":"@site/blog/2025-11-14-the-new-lab/index.md","title":"Introducing The Lab","description":"Introducing the new The Lab platform","date":"2025-11-14T00:00:00.000Z","tags":[{"inline":true,"label":"data","permalink":"/posts/tags/data"},{"inline":true,"label":"lab","permalink":"/posts/tags/lab"}],"readingTime":6.69,"hasTruncateMarker":false,"authors":[{"name":"samcm","title":"DevOps Engineer","bio":"DevOps Engineer","url":"https://github.com/samcm","github":"https://github.com/samcm","t

1witter":"https://x.com/samcmAU","imageURL":"/img/team/samcm.jpeg","key":"samcm","page":null},{"name":"savid \ud83d\ude82","title":"DevOps Engineer","bio":"DevOps Engineer","url":"https://github.com/savid","github":"https://github.com/savid","twitter":"https://x.com/savid","imageURL":"/img/team/savid.jpeg","key":"savid","page":null},{"name":"matty","title":"DevOps Engineer","bio":"DevOps Engineer","url":"https://github.com/mattevans","github":"https://github.com/mattevans","twitter":"https://x.com/mattevansnz","imageURL":"/img/team/matty.jpeg","key":"matty","page":null}],"frontMatter":{"slug":"the-lab","title":"Introducing The Lab","authors":["samcm","savid","matty"],"description":"Introducing the new The Lab platform","tags":["data","lab"],"image":"/img/blog/the-new-lab/header.png","relatedLinks":[{"name":"The Lab","url":"https://lab.ethpandaops.io/"}]},"unlisted":false,"prevItem":{"title":"Announcing Live Custody Monitoring for PeerDAS","permalink":"/posts/live-custody-monitoring-peerdas"},"nextItem":{"title":"Fusaka bandwidth estimation","permalink":"/posts/fusaka-bandwidth-estimation"}}')}}]);