https://www.tigrisdata.com/blog/assets/js/833982e2.caf82552.js

1"use strict";(globalThis.webpackChunkblog=globalThis.webpackChunkblog||[]).push([[295],{5739(e){e.exports=JSON.parse('{"permalink":"/blog/s3-conditional-writes","source":"@site/blog/2024-08-23-s3-conditional-writes/index.mdx","title":"What\u2019s the Big Deal with Conditional Writes Support in S3?","description":"There has been a lot of enthusiasm about the recent announcement by AWS about support for Conditional Writes in S3. We will dive into why this is a big deal and what it means for developers. We will also explore CAS operations and how they relate to S3 Conditional Writes feature. Finally, we will discuss whether developers can now avoid client-side consensus mechanisms completely when using S3.\\n","date":"2024-08-23T00:00:00.000Z","tags":[{"inline":true,"label":"Engineering","permalink":"/blog/tags/engineering"},{"inline":true,"label":"object storage","permalink":"/blog/tags/object-storage"},{"inline":true,"label":"conditional writes","permalink":"/blog/tags/conditional-writes"},{"inline":true,"label":"distributed systems","permalink":"/blog/tags/distributed-systems"}],"readingTime":4.43,"hasTruncateMarker":true,"authors":[{"name":"Ovais Tariq","title":"Co-Founder @ Tigris Data","url":"https://www.linkedin.com/in/ovaistariq/","imageURL":"https://github.com/ovaistariq.png","key":"ot","page":null}],"frontMatter":{"slug":"s3-conditional-writes","title":"What\u2019s the Big Deal with Conditional Writes Support in S3?","description":"There has been a lot of enthusiasm about the recent announcement by AWS about support for Conditional Writes in S3. We will dive into why this is a big deal and what it means for developers. We will also explore CAS operations and how they relate to S3 Conditional Writes feature. Finally, we will discuss whether developers can now avoid client-side consensus mechanisms completely when using S3.\\n","image":"./twin-formula-1-race-car-drivers-navigating-a-sharp-turn-on-a-track-that-weaves-through-the-bustling--58786778.jpeg","keywords":["object storage","conditional writes","cas","distributed systems"],"authors":["ot"],"tags":["Engineering","object storage","conditional writes","distributed systems"]},"unlisted":false,"prevItem":{"title":"Snip, Snap, Serve: Effortless Global Image Delivery with Tigris","permalink":"/blog/effortless-global-image-delivery"},"nextItem":{"title":"Metadata Querying for Object Storage feat. Elixir","permalink":"/blog/metadata-quering-with-elixir"}}')},11939(e,n,t){t.d(n,{A:()=>i});const i=t.p+"assets/images/twin-formula-1-race-car-drivers-navigating-a-sharp-turn-on-a-track-that-weaves-through-the-bustling--58786778-49e3f1a000cd002d19689a4ca722818d.jpeg"},12833(e,n,t){t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>l,default:()=>u,frontMatter:()=>r,metadata:()=>i,toc:()=>h});var i=t(5739),o=t(74848),a=t(28453),s=t(11939);const r={slug:"s3-conditional-writes",title:"What\u2019s the Big Deal with Conditional Writes Support in S3?",description:"There has been a lot of enthusiasm about the recent announcement by AWS about support for Conditional Writes in S3. We will dive into why this is a big deal and what it means for developers. We will also explore CAS operations and how they relate to S3 Conditional Writes feature. Finally, we will discuss whether developers can now avoid client-side consensus mechanisms completely when using S3.\n",image:"./twin-formula-1-race-car-drivers-navigating-a-sharp-turn-on-a-track-that-weaves-through-the-bustling--58786778.jpeg",keywords:["object storage","conditional writes","cas","distributed systems"],authors:["ot"],tags:["Engineering","object storage","conditional writes","distributed systems"]},l="What's the Big Deal with Conditional Writes Support in S3?",c={image:t(39601).A,authorsImageUrls:[void 0]},h=[{value:"It's all about concurrency and races.",id:"its-all-about-concurrency-and-races",level:2},{value:"Generally, concurrent writes are handled by employing mechanisms like Compare and Swap (CAS) to ensure data consistency.",id:"generally-concurrent-writes-are-handled-by-employing-mechanisms-like-compare-and-swap-cas-to-ensure-data-consistency",level:2},{value:"But without support for CAS in S3, developers have to do the heavy lifting.",id:"but-without-support-for-cas-in-s3-developers-have-to-do-the-heavy-lifting",level:2},{value:"Client-Side Locking or Coordination Mechanisms",id:"client-side-locking-or-coordination-mechanisms",level:3},{value:"Versioning",id:"versioning",level:3},{value:"Creating Middleware or API Wrappers",id:"creating-middleware-or-api-wrappers",level:3},{value:"Hence, the reason for enthusiasm with the recently announced support for conditional writes in S3.",id:"hence-the-reason-for-enthusiasm-with-the-recently-announced-support-for-conditional-writes-in-s3",level:2}

1,{value:"Do I really not need any client-side consensus mechanisms now?",id:"do-i-really-not-need-any-client-side-consensus-mechanisms-now",level:2}];function d(e){const n={a:"a",em:"em",h2:"h2",h3:"h3",li:"li",p:"p",ul:"ul",...(0,a.R)(),...e.components};return(0,o.jsxs)(o.Fragment,{children:[(0,o.jsx)("img",{src:s.A,className:"hero-image",alt:"Twin Formula 1 race car drivers navigating a sharp turn"}),"\n",(0,o.jsxs)(n.p,{children:["Twitter has been lit (well, not exactly lit, but let's say there is a lot of\nenthusiasm) with posts about the recent announcement by AWS about support for\n",(0,o.jsx)(n.a,{href:"https://aws.amazon.com/about-aws/whats-new/2024/08/amazon-s3-conditional-writes/",children:"Conditional Writes in S3"}),".\nSo I thought I would write something about it as well."]}),"\n",(0,o.jsx)(n.h2,{id:"its-all-about-concurrency-and-races",children:"It's all about concurrency and races."}),"\n",(0,o.jsx)(n.p,{children:"In distributed environments, it\u2019s common for multiple clients to attempt to\nupdate or write to the same object or dataset simultaneously. This creates a\nchallenge: ensuring that these concurrent operations do not interfere with each\nother or lead to data inconsistencies. For instance, in collaborative\nenvironments where users are simultaneously editing shared documents or\nuploading files to a common storage, there is risk of one user\u2019s actions\noverwriting another\u2019s. Without a robust mechanism to manage these concurrent\nwrites, you can end up with inconsistent data or lost updates, leading to\npotential failures in the application. Managing this concurrency requires\ncareful coordination to ensure that all operations are correctly sequenced and\nthat no data is lost or incorrectly overwritten."}),"\n",(0,o.jsx)(n.h2,{id:"generally-concurrent-writes-are-handled-by-employing-mechanisms-like-compare-and-swap-cas-to-ensure-data-consistency",children:"Generally, concurrent writes are handled by employing mechanisms like Compare and Swap (CAS) to ensure data consistency."}),"\n",(0,o.jsx)(n.p,{children:"CAS is an atomic operation that allows a system to update a value only if it\nmatches an expected current value, effectively preventing race conditions where\nmultiple transactions or threads might try to modify the same data\nsimultaneously. For example, in a scenario where multiple users attempt to\nupdate a row in a database table, CAS ensures that each update is applied only\nif the row hasn\u2019t been altered by another operation in the meantime. This\napproach enables systems to manage high levels of concurrency without resorting\nto locking mechanisms, which can be less efficient."}),"\n",(0,o.jsx)(n.h2,{id:"but-without-support-for-cas-in-s3-developers-have-to-do-the-heavy-lifting",children:"But without support for CAS in S3, developers have to do the heavy lifting."}),"\n",(0,o.jsx)(n.p,{children:"Traditionally, developers using S3 have had to employ strategies with varying\nlevels of complexity to manage challenges that come with concurrent writes. Some\nof the most common strategies include:"}),"\n",(0,o.jsx)(n.h3,{id:"client-side-locking-or-coordination-mechanisms",children:"Client-Side Locking or Coordination Mechanisms"}),"\n",(0,o.jsx)(n.p,{children:"Devs sometimes implement client-side locking mechanisms or coordination\nprotocols to ensure that only one client could write to a particular object at a\ntime. This involves using a centralized lock manager or employing distributed\nsystems like Zookeeper, etcd, or DynamoDB."}),"\n",(0,o.jsx)(n.h3,{id:"versioning",children:"Versioning"}),"\n",(0,o.jsx)(n.p,{children:"Another popular choice is utilizing S3\u2019s object versioning feature, which allows\nmultiple versions of the same object to exist. This approach helps prevent data\nloss but requires additional logic to resolve conflicts, such as merging changes\nor selecting the correct version manually."}),"\n",(0,o.jsx)(n.h3,{id:"creating-middleware-or-api-wrappers",children:"Creating Middleware or API Wrappers"}),"\n",(0,o.jsx)(n.p,{children:"Yet another popular choice is creating custom middleware or API wrappers that\nencapsulate the S3 operations with logic for handling concurrency using one of\nthe above-mentioned strategies. This additional layer manages concurrency\nguards, pre-checks, retries, and error handling in a standardized way across\nmany services. While this helps centralize the conflict resolution logic, it\nstill requires custom development and adds complexity."}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.em,{children:"These methods, while functional, result in increased code complexity, higher\nlatencies, and bugs due to potential race conditions, making the management of\nconcurrent operations with S3 a non-trivial task."})}),"\n",(0,o.jsx)(n.h2,{id:"hence-the-reason-for-enthusiasm-with-the-recently-announced-support-for-conditional-writes-in-s3",children:"Hence, the reason for enthusiasm with the recently announced support for conditional writes in S3."}),"\n",(0,o.jsxs)(n.p,{children:["Now that it is clear that handling concurrent writes at the application level is\nnon-trivial, there is good reason to be pleased with the recently announced\nsupport for\n",(0,o.jsx)(n.a,{href:"https://aws.amazon.com/about-aws/whats-new/2024/08/amazon-s3-conditional-writes/",children:"conditional writes in S3"}),"."]}),"\n",(0,o.jsx)(n.p,{children:"This new feature allows you to conditionally write or upload objects, using\nPutObject or CompleteMultipartUpload API requests, based on whether the object\nalready exists or not. This pushes the complexity of handling concurrency to S3\nin sce

1narios where multiple clients are writing to the same storage bucket, and\nyou want to avoid overwriting existing data."}),"\n",(0,o.jsx)(n.p,{children:(0,o.jsx)(n.em,{children:"Devs can throw out a lot of the concurrency related complexity now!"})}),"\n",(0,o.jsx)(n.h2,{id:"do-i-really-not-need-any-client-side-consensus-mechanisms-now",children:"Do I really not need any client-side consensus mechanisms now?"}),"\n",(0,o.jsx)(n.p,{children:"The Conditional Writes feature in S3 is primarily designed to prevent the\noverwriting of objects, ensuring that an object is only written if it doesn\u2019t\nalready exist."}),"\n",(0,o.jsx)(n.p,{children:"While S3 Conditional Writes feature and CAS operations share a similar purpose,\nbut they are not fully equivalent, and S3 Conditional Writes do not cover all\nthe cases supported by CAS. CAS provides a broader scope, enabling you to\ncompare and update based on the actual value of the data, not just its presence.\nThis can include complex conditions, such as:"}),"\n",(0,o.jsxs)(n.ul,{children:["\n",(0,o.jsx)(n.li,{children:"matching object content using etags,"}),"\n",(0,o.jsx)(n.li,{children:"performing object \u201cmodified date\u201d based comparisons,"}),"\n",(0,o.jsx)(n.li,{children:"comparing object tag values, and"}),"\n",(0,o.jsx)(n.li,{children:"a combination of the above."}),"\n"]}),"\n",(0,o.jsx)(n.p,{children:"So, depending on the requirements, concurrency handling on the application side\nusing one of the above-mentioned approaches may still be needed when using S3."}),"\n",(0,o.jsxs)(n.p,{children:["There are also other object storage services that provide more comprehensive CAS\nsupport such as ",(0,o.jsx)(n.a,{href:"https://azure.microsoft.com/en-us/products/storage/blobs",children:"ABS"}),",\n",(0,o.jsx)(n.a,{href:"https://cloud.google.com/storage",children:"GCS"}),", ",(0,o.jsx)(n.a,{href:"https://min.io/",children:"MinIO"}),",\n",(0,o.jsx)(n.a,{href:"https://developers.cloudflare.com/r2/",children:"R2"}),", and\n",(0,o.jsx)(n.a,{href:"https://www.tigrisdata.com/docs/objects/conditionals/",children:"Tigris"}),"."]})]})}function u(e={}){const{wrapper:n}={...(0,a.R)(),...e.components};return n?(0,o.jsx)(n,{...e,children:(0,o.jsx)(d,{...e})}):d(e)}},28453(e,n,t){t.d(n,{R:()=>s,x:()=>r});var i=t(96540);const o={},a=i.createContext(o);function s(e){const n=i.useContext(a);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(o):e.components||o:s(e.components),i.createElement(a.Provider,{value:n},e.children)}},39601(e,n,t){t.d(n,{A:()=>i});const i=t.p+"assets/images/twin-formula-1-race-car-drivers-navigating-a-sharp-turn-on-a-track-that-weaves-through-the-bustling--58786778-49e3f1a000cd002d19689a4ca722818d.jpeg"}}]);