1"use strict";(self.webpackChunkcrawlee_website=self.webpackChunkcrawlee_website||[]).push([["16278"],{62036(e,n,o){o.r(n),o.d(n,{metadata:()=>s,default:()=>w,frontMatter:()=>c,contentTitle:()=>d,toc:()=>p,assets:()=>h});var s=JSON.parse('{"id":"guides/session-management","title":"Session Management","description":"How to manage your cookies, proxy IP rotations and more","source":"@site/versioned_docs/version-3.16/guides/session_management.mdx","sourceDirName":"guides","slug":"/guides/session-management","permalink":"/js/docs/3.16/guides/session-management","draft":false,"unlisted":false,"editUrl":"https://github.com/apify/crawlee/edit/master/website/versioned_docs/version-3.16/guides/session_management.mdx","tags":[],"version":"3.16","lastUpdatedBy":"Muhammad Haadhee Sheeraz Mian","lastUpdatedAt":1790948334000,"frontMatter":{"id":"session-management","title":"Session Management","description":"How to manage your cookies, proxy IP rotations and more"},"sidebar":"docs","previous":{"title":"Proxy Management","permalink":"/js/docs/3.16/guides/proxy-management"},"next":{"title":"Scaling our crawlers","permalink":"/js/docs/3.16/guides/scaling-crawlers"}}'),t=o(62540),r=o(43023),i=o(20188),a=o(77779),l=o(98263),u=o(77683);let c={id:"session-management",title:"Session Management",description:"How to manage your cookies, proxy IP rotations and more"},d,h={},p=[];function f(e){let n={a:"a",code:"code",li:"li",p:"p",ul:"ul",...(0,r.R)(),...e.components};return(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(n.p,{children:["\u200B",(0,t.jsx)(i.A,{to:"core/class/SessionPool",children:(0,t.jsx)(n.code,{children:"SessionPool"})})," is a class that allows us to handle the rotation of proxy IP addresses along with cookies and other custom settings in Crawlee."]}),"\n",(0,t.jsx)(n.p,{children:"The main benefit of using Session pool is that we can filter out blocked or non-working proxies,\nso our actor does not retry requests over known blocked/non-working proxies.\nAnother benefit of using SessionPool is that we can store information tied tightly to an IP address,\nsuch as cookies, auth tokens, and particular headers. Having our cookies and other identifiers used only with a specific IP will reduce the chance of being blocked.\nThe last but not least benefit is the even rotation of IP addresses - SessionPool picks the session randomly,\nwhich should prevent burning out a small pool of available IPs."}),"\n",(0,t.jsxs)(n.p,{children:["Check out the ",(0,t.jsx)(n.a,{href:"./avoid-blocking",children:"avoid blocking guide"})," for more information about blocking."]}),"\n",(0,t.jsx)(n.p,{children:"Now let's take a look at the examples of how to use Session pool:"}),"\n",(0,t.jsxs)(n.ul,{children:["\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"basic-crawler/class/BasicCrawler",children:(0,t.jsx)(n.code,{children:"BasicCrawler"})}),";"]}),"\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"http-crawler/class/HttpCrawler",children:(0,t.jsx)(n.code,{children:"HttpCrawler"})}),";"]}),"\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"cheerio-crawler/class/CheerioCrawler",children:(0,t.jsx)(n.code,{children:"CheerioCrawler"})}),";"]}),"\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"jsdom-crawler/class/JSDOMCrawler",children:(0,t.jsx)(n.code,{children:"JSDOMCrawler"})}),";"]}),"\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"playwright-crawler/class/PlaywrightCrawler",children:(0,t.jsx)(n.code,{children:"PlaywrightCrawler"})}),";"]}),"\n",(0,t.jsxs)(n.li,{children:["with ",(0,t.jsx)(i.A,{to:"puppeteer-crawler/class/PuppeteerCrawler",children:(0,t.jsx)(n.code,{children:"PuppeteerCrawler"})}),";"]}),"\n",(0,t.jsx)(n.li,{children:"without a crawler (standalone usage to manage sessions manually)."}),"\n"]}),"\n",(0,t.jsxs)(a.A,{groupId:"session_pool",children:[(0,t.jsx)(l.A,{value:"basic",label:"BasicCrawler",children:(0,t.jsx)(u.A,{language:"js",children:"import { BasicCrawler, ProxyConfiguration } from 'crawlee';\nimport { gotScraping } from 'got-scraping';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new BasicCrawler({\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration.\n sessionPoolOptions: { maxPoolSize: 100 },\n async requestHandler({ request, session }) {\n const { url } = request;\n const requestOptions = {\n url,\n // We use session id in order to have the same proxyUrl\n // for all the requests using the same session.\n proxyUrl: await proxyConfiguration.newUrl(session?.id),\n throwHttpErrors: false,\n headers: {\n // If you want to use the cookieJar.\n // This way you get the Cookie headers string from session.\n Cookie
1: session?.getCookieString(url),\n },\n };\n let response;\n\n try {\n response = await gotScraping(requestOptions);\n } catch (e) {\n if (e === 'SomeNetworkError') {\n // If a network error happens, such as timeout, socket hangup, etc.\n // There is usually a chance that it was just bad luck\n // and the proxy works. No need to throw it away.\n session?.markBad();\n }\n throw e;\n }\n\n // Automatically retires the session based on response HTTP status code.\n session?.retireOnBlockedStatusCodes(response.statusCode);\n\n if (response.body.includes('You are blocked!')) {\n // You are sure it is blocked.\n // This will throw away the session.\n session?.retire();\n }\n\n // Everything is ok, you can get the data.\n // No need to call session.markGood -> BasicCrawler calls it for you.\n\n // If you want to use the CookieJar in session you need.\n session?.setCookiesFromResponse(response);\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"http",label:"HttpCrawler",children:(0,t.jsx)(u.A,{language:"js",children:"import { HttpCrawler, ProxyConfiguration } from 'crawlee';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new HttpCrawler({\n // To use the proxy IP session rotation logic, you must turn the proxy usage on.\n proxyConfiguration,\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration.\n sessionPoolOptions: { maxPoolSize: 100 },\n // Set to true if you want the crawler to save cookies per session,\n // and set the cookie header to request automatically (default is true).\n persistCookiesPerSession: true,\n async requestHandler({ session, body }) {\n const title = (body as string).match(/<title(?:.*?)>(.*?)<\\/title>/)?.[1];\n\n if (title === 'Blocked') {\n session?.retire();\n } else if (title === 'Not sure if blocked, might also be a connection error') {\n session?.markBad();\n }
1 else {\n // session.markGood() - this step is done automatically in BasicCrawler.\n }\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"cheerio",label:"CheerioCrawler",default:!0,children:(0,t.jsx)(u.A,{language:"js",children:"import { CheerioCrawler, ProxyConfiguration } from 'crawlee';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new CheerioCrawler({\n // To use the proxy IP session rotation logic, you must turn the proxy usage on.\n proxyConfiguration,\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration.\n sessionPoolOptions: { maxPoolSize: 100 },\n // Set to true if you want the crawler to save cookies per session,\n // and set the cookie header to request automatically (default is true).\n persistCookiesPerSession: true,\n async requestHandler({ session, $ }) {\n const title = $('title').text();\n\n if (title === 'Blocked') {\n session?.retire();\n } else if (title === 'Not sure if blocked, might also be a connection error') {\n session?.markBad();\n }
1 else {\n // session.markGood() - this step is done automatically in BasicCrawler.\n }\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"jsdom",label:"JSDOMCrawler",children:(0,t.jsx)(u.A,{language:"js",children:"import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new JSDOMCrawler({\n // To use the proxy IP session rotation logic, you must turn the proxy usage on.\n proxyConfiguration,\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration.\n sessionPoolOptions: { maxPoolSize: 100 },\n // Set to true if you want the crawler to save cookies per session,\n // and set the cookie header to request automatically (default is true).\n persistCookiesPerSession: true,\n async requestHandler({ session, window }) {\n const title = window.document.title;\n\n if (title === 'Blocked') {\n session?.retire();\n } else if (title === 'Not sure if blocked, might also be a connection error') {\n session?.markBad();\n }
1 else {\n // session.markGood() - this step is done automatically in BasicCrawler.\n }\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"playwright",label:"PlaywrightCrawler",children:(0,t.jsx)(u.A,{language:"js",children:"import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new PlaywrightCrawler({\n // To use the proxy IP session rotation logic, you must turn the proxy usage on.\n proxyConfiguration,\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration\n sessionPoolOptions: { maxPoolSize: 100 },\n // Set to true if you want the crawler to save cookies per session,\n // and set the cookies to page before navigation automatically (default is true).\n persistCookiesPerSession: true,\n async requestHandler({ page, session }) {\n const title = await page.title();\n\n if (title === 'Blocked') {\n session?.retire();\n } else if (title === 'Not sure if blocked, might also be a connection error') {\n session?.markBad();\n }
1 else {\n // session.markGood() - this step is done automatically in PlaywrightCrawler.\n }\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"puppeteer",label:"PuppeteerCrawler",children:(0,t.jsx)(u.A,{language:"js",children:"import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';\n\nconst proxyConfiguration = new ProxyConfiguration({\n /* opts */\n});\n\nconst crawler = new PuppeteerCrawler({\n // To use the proxy IP session rotation logic, you must turn the proxy usage on.\n proxyConfiguration,\n // Activates the Session pool (default is true).\n useSessionPool: true,\n // Overrides default Session pool configuration\n sessionPoolOptions: { maxPoolSize: 100 },\n // Set to true if you want the crawler to save cookies per session,\n // and set the cookies to page before navigation automatically (default is true).\n persistCookiesPerSession: true,\n async requestHandler({ page, session }) {\n const title = await page.title();\n\n if (title === 'Blocked') {\n session?.retire();\n } else if (title === 'Not sure if blocked, might also be a connection error') {\n session?.markBad();\n }
1 else {\n // session.markGood() - this step is done automatically in PuppeteerCrawler.\n }\n },\n});\n"})}),(0,t.jsx)(l.A,{value:"standalone",label:"Standalone",children:(0,t.jsx)(u.A,{language:"js",children:"import { SessionPool } from 'crawlee';\n\n// Override the default Session pool configuration.\nconst sessionPoolOptions = {\n maxPoolSize: 100,\n};\n\n// Open Session Pool.\nconst sessionPool = await SessionPool.open(sessionPoolOptions);\n\n// Get session.\nconst session = await sessionPool.getSession();\n\n// Increase the errorScore.\nsession.markBad();\n\n// Throw away the session.\nsession.retire();\n\n// Lower the errorScore and mark the session good.\nsession.markGood();\n"})})]}),"\n",(0,t.jsx)(n.p,{children:"These are the basics of configuring SessionPool.\nPlease, bear in mind that a Session pool needs time to find working IPs and build up the pool,\nso we will probably see a lot of errors until it becomes stabilized."})]})}function w(e={}){let{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,t.jsx)(n,{...e,children:(0,t.jsx)(f,{...e})}):f(e)}},98263(e,n,o){o.d(n,{A:()=>a});var s=o(62540);o(63696);var t=o(11750),r=o(23865);function i({children:e,className:n,hidden:o}){return(0,s.jsx)("div",{role:"tabpanel",className:(0,t.A)("tabItem_wHwb",n),hidden:o,children:e})}function a({children:e,className:n,value:o}){let{selectedValue:t,lazy:l}=(0,r.uc)(),u=o===t;return!u&&l?null:(0,s.jsx)(i,{className:n,hidden:!u,children:e})}},77779(e,n,o){o.d(n,{A:()=>h});var s=o(62540);o(63696);var t=o(11750),r=o(46065),i=o(23865),a=o(45194),l=o(83029);function u({className:e}){let{selectedValue:n,selectValue:o,tabValues:r,block:l}=(0,i.uc)(),c=[],{blockElementScrollPositionUntilNextRender:d}=(0,a.a_)(),h=e=>{let s=e.currentTarget,t=r[c.indexOf(s)].value;t!==n&&(d(s),o(t))},p=e=>{let n=null;switch(e.key){case"Enter":h(e);break;case"ArrowRight":{let o=c.indexOf(e.currentTarget)+1;n=c[o]??c[0];break}case"ArrowLeft":{let o=c.indexOf(e.currentTarget)-1;n=c[o]??c[c.length-1]}}n?.focus()};return(0,s.jsx)("ul",{role:"tablist","aria-orientation":"horizontal",className:(0,t.A)("tabs",{"tabs--block":l},e),children:r.map(({value:e,label:o,attributes:r})=>(0,s.jsx)("li",{role:"tab",tabIndex:n===e?0:-1,"aria-selected":n===e,ref:e=>{c.push(e)},onKeyDown:p,onClick:h,...r,className:(0,t.A)("tabs__item","tabItem_l0OV",r?.className,{"tabs__item--active":n===e}),children:o??e},e))})}function c({children:e}){return(0,s.jsx)("div",{className:"margin-top--md",children:e})}function d({className:e,children:n}){return(0,s.jsxs)("div",{className:(0,t.A)(r.G.tabs.container,"tabs-container","tabList_J5MA"),children:[(0,s.jsx)(u,{className:e}),(0,s.jsx)(c,{children:n})]})}function h(e){let n=(0,l.A)(),o=(0,i.OC)(e);return(0,s.jsx)(i.O_,{value:o,children:(0,s.jsx)(d,{className:e.className,children:(0,i.vT)(e.children)})},String(n))}},23865(e,n,o){o.d(n,{OC:()=>h,O_:()=>w,uc:()=>f,vT:()=>c});var s=o(62540),t=o(63696),r=o(49519),i=o(6743),a=o(88511),l=o(15396),u=o(89030);function c(e){return t.Children.toArray(e).filter(e=>"\n"!==e)}function d({value:e,tabValues:n}){return n.some(n=>n.value===e)}function h(e){let n,{defaultValue:o,queryString:s=!1,groupId:c}=e,h=function(e){let{values:n,children:o}=e;return(0,t.useMemo)(()=>{let e=n??t.Children.toArray(o).flatMap(e=>{if(!e)return[];if((0,t.isValidElement)(e)&&function(e){let{props:n}=e;return!!n&&"object"==typeof n&&"value"in n}(e))return[e];let n="string"==typeof e.type?e.type:e.type.name;throw Error(`Docusaurus error: Bad <Tabs> child <${n}>: all children of the <Tabs> component should be <TabItem>, and every <TabItem> should have a unique "value" prop. 2If you do not want to pass on a "value" prop to the direct children of <Tabs>, you can also pass an explicit <Tabs values={...}> prop.`)}).map(({props:{value:e,label:n,attributes:o,default:s}})=>({value:e,label:n,attributes:o,default:s})),s=(0,l.XI)(e,(e,n)=>e.value===n.value);if(s.length>0)throw Error(`Docusaurus error: Duplicate values "${s.map(e=>`'${e.value}'`).join(", ")}" found in <Tabs>. Every value needs to be unique.`);return e},[n,o])}(e),[p,f]=(0,t.useState)(()=>(function({defaultValue:e,tabValues:n}){if(0===n.length)throw Error("Docusaurus error: the <Tabs> component requires at least one <TabItem> children component");if(e){if(!d({value:e,tabValues:n}))throw Error(`Docusaurus error: The <Tabs> has a defaultValue "${e}" but none of its children has the corresponding value. Available values are: ${n.map(e=>e.value).join(", ")}. If you intend to show no default tab, use defaultValue={null} instead.`);return e}let o=n.find(e=>e.default)??n[0];if(!o)throw Error("Unexpected error: 0 tabValues");return o.value})({defaultValue:o,tabValues:h})),[w,m]=function({queryString:e=!1,groupId:n}){let o=(0,r.W6)(),s=function({queryString:e=!1,groupId:n}){if("string"==typeof e)return e;if(!1===e)return null;if(!0===e&&!n)throw Error('Docusaurus error: The <Tabs> component groupId prop is required if queryString=true, because this value is used as the search param name. You can also provide an explicit value such as queryString="my-search-param".');return n??null}({queryString:e,groupId:n});return[(0,a.aZ)(s),(0,t.useCallback)(e=>{if(!s)return;let n=new URLSearchParams(o.location.search);n.set(s,e),o.replace({...o.location,search:n.toString()})},[s,o])]}({queryString:s,groupId:c}),[g,x]=function({groupId:e}){let n=e?`docusaurus.tab.${e}`:null,[o,s]=(0,u.Dv)(n);return[o,(0,t.useCallback)(e=>{n&&s.set(e)},[n,s])]}({groupId:c}),y=d({value:n=w??g,tabValues:h})?n:null;return(0,i.A)(()=>{y&&f(y)},[y]),{selectedValue:p,selectValue:(0,t.useCallback)(e=>{if(!d({value:e,tabValues:h}))throw Error(`Can't select invalid tab value=${e}`);f(e),m(e),x(e)},[m,x,h]),tabValues:h,lazy:e.lazy??!1,block:e.block??!1}}let p=(0,t.createContext)(null);function f(){let e=t.useContext(p);if(!e)throw Error("useTabsContext() must be used within a Tabs component");return e}function w(e){return(0,s.jsx)(p.Provider,{value:e.value,children:e.children})}},20188(e,n,o){o.d(n,{A:()=>l});var s=o(62540);o(63696);var t=o(12212),r=o(17607),i=o(73228);let a=o(6715)[0],l=({to:e,children:n})=>{let o=(0,r.r)(),{siteConfig:l}=(0,i.A)();return l.presets[0][1].docs.disableVersioning||o.version===a?(0,s.jsx)(t.A,{to:`/js/api/${e}`,children:n}):(0,s.jsx)(t.A,{to:`/js/api/${"current"===o.version?"next":o.version}/${e}`,children:n})}},43023(e,n,o){o.d(n,{R:()=>i,x:()=>a});var s=o(63696);let t={},r=s.createContext(t);function i(e){let n=s.useContext(r);return s.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(t):e.components||t:i(e.components),s.createElement(r.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.