https://calv.info/_next/static/chunks/pages/who-s-who-in-upstream-caching-955ada1765324a21.js

1(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[869],{9516:(e,s,n)=>{(window.__NEXT_P=window.__NEXT_P||[]).push(["/who-s-who-in-upstream-caching",function(){return n(9579)}])},9579:(e,s,n)=>{"use strict";n.r(s),n.d(s,{__N_SSG:()=>t,blog:()=>r,date:()=>i,default:()=>d,excerpt:()=>h,title:()=>c});var a=n(7876),l=n(9636),t=!0;let r=!0,i=new Date("2010-08-24T23:14:14.956Z"),h="Today, that subject is upstream caching, hints in the HTTP headers that developers can give to routers and content delivery systems that intermediate between client and server.",c="Who’s who in upstream caching";function o(e){let s={a:"a",code:"code",img:"img",p:"p",pre:"pre",span:"span",...(0,l.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(s.p,{children:"Well, once again there is too much other stuff to work on and my last blog project didn't turn out quite the way I hoped, so there hasn't been a recent blog post. Look for a few concerning my latest project in the coming weeks, but for now I'll be putting up a few shorter or less technical ones about various subjects that I'd like to brain dump. Today, that subject is upstream caching, hints in the HTTP headers that developers can give to routers and content delivery systems that intermediate between client and server."}),"\n",(0,a.jsxs)(s.p,{children:["Given a particular website or web service, the original developer generally has the best idea of how often different sorts of information are updated on the site. It might be the case that the content is updated in a matter of\xa0",(0,a.jsx)(s.a,{href:"http://www.facebook.com/",children:"minutes"}),",\xa0",(0,a.jsx)(s.a,{href:"http://www.woot.com/",children:"days"}),", or maybe\xa0",(0,a.jsx)(s.a,{href:"https://calv.info",children:"once in a blue moon"}),". At any rate, it can be pretty costly for the server's\xa0bandwidth\xa0if it constantly has to send the client information that the client has already seen. Therefore at any point in between the connection between server and client, whether it is through a proxy or a router, there may be some caching done to save bandwidth. But for a website serving private information, the possibility of a shared cache presents a clear risk to the privacy and security of the service. Therefore, services can add\xa0",(0,a.jsx)(s.a,{href:"http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9",children:"directives to the HTTP headers"}),"\xa0in a response to determine how the response should be cached."]}),"\n",(0,a.jsxs)(s.p,{children:["I was curious about these particular headers and how often they are used in the wild. Thanks to\xa0",(0,a.jsx)(s.a,{href:"http://www.alexa.com/topsites",children:"alexa"}),", I was able to download a CSV of the top 1 million websites accessed by their users. As a sidenote, that is really awesome that Alexa makes this data available in a developer-friendly format, way to go guys! At any rate, it was time to test my theory that websites at greater scale (more popular) would be more likely to have customized this upstream cache, particularly to include values for a stale timeout. To do this, I created a simple script to return the cache-control header from a request:"]}),"\n",(0,a.jsx)(s.pre,{children:(0,a.jsxs)(s.code,{className:"hljs language-python",children:[(0,a.jsxs)(s.span,{className:"hljs-function",children:[(0,a.jsx)(s.span,{className:"hljs-keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"hljs-title",children:"check_cache"}),"(",(0,a.jsx)(s.span,{className:"hljs-params",children:"url"}),"):"]}),"\n req = urllib2.Request(url)\n req.add_header(",(0,a.jsx)(s.span,{className:"hljs-string",children:"'User-Agent'"}),", ",(0,a.jsx)(s.span,{className:"hljs-string",children:"'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.126 Safari/533.4'"}),")\n opener = urllib2.build_opener()\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"try"}),":\n  resp = opener.",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"open"}),"(req)\n  ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"if"})," ",(0,a.jsx)(s.span,{className:"hljs-string",children:"'cache-control'"})," ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," resp.info().",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"dict"}),":\n   ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"return"})," resp.info().",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"dict"}),"[",(0,a.jsx)(s.span,{className:"hljs-string",children:"'cache-control'"}),"].replace(",(0,a.jsx)(s.span,{className:"hljs-string",children:'" "'}),", ",(0,a.jsx)(s.span,{className:"hljs-string",children:'""'}),").split(",(0,a.jsx)(s.span,{className:"hljs-string",children:'","'}),")\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"except"})," Exception:\n  ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"return"})," []\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"return"})," [",(0,a.jsx)(s.span,{className:"hljs-string",children:"'none'"}),"]\n"]})}),"\n",(0,a.jsxs)(s.p,{children:["The reason for the spoofed user agent header is that\xa0",(0,a.jsx)(s.a,{href:"http://www.wikipedia.org/",children:"some popular websites"}),"\xa0block python's native user agent header. Some websites might also not have a public facing homepage, so for those we will not assume anything. For everything else that does not have a cache-control header, we'll return a list of 'none'."]}),"\n",(0,a.jsxs)(s.p,{children:["Well, we have our list of top websites from Alexa and a method to get individual keys for a website. The way I broke up the findings was by taking the websites 50 at a time and simply adding together their key counts. Admittedly there are a few caveats to this method, one is that websites operated by a single company under different c

1ountries show up as\xa0",(0,a.jsx)(s.a,{href:"http://google.fr/",children:"different domains"}),", even though it is likely that they use the same caching policy."]}),"\n",(0,a.jsx)(s.p,{children:"To run the script, I used the run method which gathered results for every 50 websites."}),"\n",(0,a.jsx)(s.pre,{children:(0,a.jsxs)(s.code,{className:"hljs language-python",children:["CHUNK_SIZE = ",(0,a.jsx)(s.span,{className:"hljs-number",children:"50"}),"\nCHUNKS = ",(0,a.jsx)(s.span,{className:"hljs-number",children:"20"}),"\n\n",(0,a.jsxs)(s.span,{className:"hljs-function",children:[(0,a.jsx)(s.span,{className:"hljs-keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"hljs-title",children:"get_next_chunk"}),"(",(0,a.jsx)(s.span,{className:"hljs-params",children:"f"}),"):"]}),"\n cache_freq = defaultdict(",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"int"}),")\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," i ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"range"}),"(CHUNK_SIZE):\n  s = f.readline()\n  url = s.split(",(0,a.jsx)(s.span,{className:"hljs-string",children:'","'}),")[",(0,a.jsx)(s.span,{className:"hljs-number",children:"1"}),"].strip()\n  ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"print"})," ",(0,a.jsx)(s.span,{className:"hljs-string",children:'"Reading:"'}),", url\n  ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," check_cache(",(0,a.jsx)(s.span,{className:"hljs-string",children:'"http://"'})," + url):\n   cache_freq[key.split(",(0,a.jsx)(s.span,{className:"hljs-string",children:'"="'}),")[",(0,a.jsx)(s.span,{className:"hljs-number",children:"0"}),"]] += ",(0,a.jsx)(s.span,{className:"hljs-number",children:"1"}),"\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"return"})," cache_freq\n\n\n",(0,a.jsxs)(s.span,{className:"hljs-function",children:[(0,a.jsx)(s.span,{className:"hljs-keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"hljs-title",children:"run"}),"(",(0,a.jsxs)(s.span,{className:"hljs-params",children:["skip = ",(0,a.jsx)(s.span,{className:"hljs-number",children:"0"})]}),"):"]}),"\n f = ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"open"}),"(",(0,a.jsx)(s.span,{className:"hljs-string",children:"'/Users/Calvin/Downloads/top-1m.csv'"}),", ",(0,a.jsx)(s.span,{className:"hljs-string",children:"'r'"}),")\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," i ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"range"}),"(skip):\n  f.readline()\n results = []\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," i ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"range"}),"(CHUNKS):\n  freq = get_next_chunk(f)\n  results.append(freq)\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"return"})," results\n"]})}),"\n",(0,a.jsx)(s.p,{children:"Add in a nice simple plotting script and we're ready to see the results:"}),"\n",(0,a.jsx)(s.pre,{children:(0,a.jsxs)(s.code,{className:"hljs language-python",children:[(0,a.jsx)(s.span,{className:"hljs-keyword",children:"import"})," matplotlib.pyplot ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"as"})," plt\n",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"import"})," numpy, random\n\n",(0,a.jsxs)(s.span,{className:"hljs-function",children:[(0,a.jsx)(s.span,{className:"hljs-keyword",children:"def"})," ",(0,a.jsx)(s.span,{className:"hljs-title",children:"plot"}),"(",(0,a.jsx)(s.span,{className:"hljs-params",children:"results"}),"):"]}),"\n ind = numpy.arange(CHUNKS)\n width = ",(0,a.jsx)(s.span,{className:"hljs-number",children:"0.1"}),"\n fig = plt.figure()\n p = fig.add_subplot(",(0,a.jsx)(s.span,{className:"hljs-number",children:"111"}),")\n\n ",(0,a.jsx)(s.span,{className:"hljs-comment",children:"# Define lambda functions for map/reduce"}),"\n key_map = ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"lambda"}

1)," x: x.keys()\n key_reduce = ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"lambda"})," x,y: ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"set"}),"(x) & ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"set"}),"(y)\n unique_keys = reduce(key_reduce, ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"map"}),"(key_map, results))\n\n colors = ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"dict"}),"([(key, (random.random(),\n        random.random(),\n        random.random())) ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," unique_keys])\n legend = ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"dict"}),"()\n\n ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," unique_keys:\n  key_counts = [result[key] ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," result ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," results]\n  legend[key] = p.plot(key_counts, color=colors[key], lw=",(0,a.jsx)(s.span,{className:"hljs-number",children:"4.0"}),")\n\n p.legend(",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"tuple"}),"([legend[key][",(0,a.jsx)(s.span,{className:"hljs-number",children:"0"}),"] ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," unique_keys]), ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"tuple"}),"([key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," key ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," unique_keys]))\n p.set_title(",(0,a.jsx)(s.span,{className:"hljs-string",children:"'Cache-Control Keys by Top Sites'"}),")\n p.set_ylabel(",(0,a.jsx)(s.span,{className:"hljs-string",children:"'Frequency of Key'"}),")\n p.set_xlabel(",(0,a.jsx)(s.span,{className:"hljs-string",children:"'Website Groupings'"}),")\n p.set_xticks(ind+",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"len"}),"(unique_keys) * width / ",(0,a.jsx)(s.span,{className:"hljs-number",children:"2.0"}),")\n p.set_xticklabels(",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"tuple"}),"([",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"str"}),"(i) ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"for"})," i ",(0,a.jsx)(s.span,{className:"hljs-keyword",children:"in"})," ",(0,a.jsx)(s.span,{className:"hljs-built_in",children:"range"}),"(",(0,a.jsx)(s.span,{className:"hljs-number",children:"1050"}),")[",(0,a.jsx)(s.span,{className:"hljs-number",children:"50"}),"::",(0,a.jsx)(s.span,{className:"hljs-number",children:"50"}),"]]))\n plt.show()\n"]})}),"\n",(0,a.jsx)(s.p,{children:"There we have it, let's check out the results of the caching for the top 100 websites:"}),"\n",(0,a.jsx)(s.p,{children:(0,a.jsx)(s.img,{src:"https://assets.contents.io/asset_5tdmb4YE.png",alt:""})}),"\n",(0,a.jsxs)(s.p,{children:["A few things become clear here, although the granularity is a bit rough. If I wanted to wait through querying 1000 websites again I might have increased the amount of saved data. At any rate, from this graph we can see a distinct downward trend in the max-age key and the private key, keys which regulate how long items remain in the cache and whether they can be shared among connections. The sites with keys also drops, as we see the 'none' key steadily rising. Clearly the sites needing the most performance take the most care in their upstream caching while those that can afford to be more lax do not take these caching measures. It's interesting to see breakdowns in individual sites -\xa0",(0,a.jsx)(s.a,{href:"http://amazon.com/",children:"Amazon"}),"\xa0is a notable exception to this rule, a site without an outbound cac

1hing policy despite its size and scale. Perhaps they are doing something more interesting by specifying a no-cache policy if a user is authenticated, but intentionally leaving anonymous users an optimized site experience."]})]})}function d(e={}){let{wrapper:s}={...(0,l.R)(),...e.components};return s?(0,a.jsx)(s,{...e,children:(0,a.jsx)(o,{...e})}):o(e)}},9636:(e,s,n)=>{"use strict";n.d(s,{R:()=>r});var a=n(7876),l=n(8230),t=n.n(l);function r(e){return{a:({children:e,href:s})=>(0,a.jsx)(t(),{href:s,children:e}),...e}}}},e=>{e.O(0,[636,6593,8792],()=>e(e.s=9516)),_N_E=e.O()}]);