https://previewchecks.com/guides/crawler-access

1<!DOCTYPE html><html  lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Crawler access - PreviewChecks guide</title><link rel="stylesheet" href="/_nuxt/entry.-PFM0SpN.css" crossorigin><meta name="description" content="How to keep social preview crawlers from being blocked by WAF, challenges, strict bot rules or image protections while public preview metadata stays crawlable."><meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1"><meta name="author" content="CoMStudio OÜ"><meta name="application-name" content="PreviewChecks"><meta name="theme-color" content="#4f46e5"><meta name="color-scheme" content="light dark"><meta name="mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-title" content="PreviewChecks"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta property="og:type" content="website"><meta property="og:site_name" content="PreviewChecks"><meta property="og:locale" content="en_US"><meta property="og:url" content="https://previewchecks.com/guides/crawler-access"><meta property="og:title" content="Crawler access - PreviewChecks guide"><meta property="og:description" content="How to keep social preview crawlers from being blocked by WAF, challenges, strict bot rules or image protections while public preview metadata stays crawlable."><meta property="og:image" content="https://previewchecks.com/og-image.png"><meta property="og:image:secure_url" content="https://previewchecks.com/og-image.png"><meta property="og:image:type" content="image/png"><meta property="og:image:width" content="1200"><meta property="og:image:height" content="630"><meta property="og:image:alt" content="PreviewChecks social preview debugger card"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="Crawler access - PreviewChecks guide"><meta name="twitter:description" content="How to keep social preview crawlers from being blocked by WAF, challenges, strict bot rules or image protections while public preview metadata stays crawlable."><meta name="twitter:image" content="https://previewchecks.com/og-image.png"><meta name="twitter:image:type" content="image/png"><meta name="twitter:image:width" content="1200"><meta name="twitter:image:height" content="630"><meta name="twitter:image:alt" content="PreviewChecks social preview debugger card"><link rel="manifest" href="/site.webmanifest?v=1"><link rel="icon" type="image/svg+xml" href="/favicon-v1.svg"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32-v1.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16-v1.png"><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon-v1.png"><link rel="canonical" href="https://previewchecks.com/guides/crawler-access">

1<script type="application/ld+json" data-hid="previewchecks-guide-crawler-access-structured-data">{"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://comstud.io/#organization","name":"CoMStudio OÜ","url":"https://comstud.io/"},{"@type":"WebSite","@id":"https://previewchecks.com/#website","name":"PreviewChecks","url":"https://previewchecks.com/","inLanguage":"en","publisher":{"@id":"https://comstud.io/#organization"}},{"@type":"WebPage","@id":"https://previewchecks.com/guides/crawler-access#webpage","name":"Crawler access - PreviewChecks guide","description":"How to keep social preview crawlers from being blocked by WAF, challenges, strict bot rules or image protections while public preview metadata stays crawlable.","url":"https://previewchecks.com/guides/crawler-access","inLanguage":"en","isPartOf":{"@id":"https://previewchecks.com/#website"},"publisher":{"@id":"https://comstud.io/#organization"}}]}</script>

1<script>"use strict";(()=>{const o=window,e=document.documentElement,c=["dark","light"],s=getStorageValue("localStorage","nuxt-color-mode")||"system";let r=s==="system"?f():s;const l=e.getAttribute("data-color-mode-forced");l&&(r=l),i(r),o["__NUXT_COLOR_MODE__"]={preference:s,value:r,getColorScheme:f,addColorScheme:i,removeColorScheme:d};function i(t){const a=""+t+"",n="";e.classList?e.classList.add(a):e.className+=" "+a,n&&e.setAttribute("data-"+n,t)}function d(t){const a=""+t+"",n="";e.classList?e.classList.remove(a):e.className=e.className.replace(new RegExp(a,"g"),""),n&&e.removeAttribute("data-"+n)}function u(t){return o.matchMedia("(prefers-color-scheme"+t+")")}function f(){if(o.matchMedia&&u("").media!=="not all"){for(const t of c)if(u(":"+t).matches)return t}return"light"}})();function getStorageValue(o,e){switch(o){case"localStorage":try{return window.localStorage.getItem(e)}catch{return null}case"sessionStorage":try{return window.sessionStorage.getItem(e)}catch{return null}case"cookie":try{return getCookie(e)}catch{return null}default:return null}}function getCookie(o){const c=("; "+window.document.cookie).split("; "+o+"=");if(c.length===2){const s=c.pop();return s?s.split(";").shift():null}}</script>

1</head><body><div id="__nuxt"><main class="bg-base-200 text-base-content min-h-screen"><article class="mx-auto w-full max-w-3xl px-4 py-12 sm:px-6 lg:py-16"><a class="link link-hover text-sm font-semibold" href="/">PreviewChecks</a><h1 class="mt-4 text-4xl font-black tracking-tight text-balance"> Keep preview crawlers out of challenges </h1><p class="text-base-content/70 mt-5 text-lg leading-8"> A page can look normal in your browser while a link-preview crawler receives a challenge, a 403 response or a login wall. Test the public response first, then change only the rule that blocks legitimate preview traffic. </p><section class="panel mt-8 p-5 sm:p-6"><h2 class="text-2xl font-black tracking-tight">What a preview crawler needs</h2><ul class="text-base-content/75 mt-4 list-disc space-y-2 pl-5 leading-7"><li> A public URL that returns the intended HTML without a cookie or signed-in session. </li><li>Metadata in the server response, not only after client-side JavaScript runs.</li><li>Public image URLs with a successful response and an image MIME type.</li><li>A redirect chain that stays reachable under the same security policy.</li><li>No interactive challenge, consent wall or geofence in front of the metadata.</li></ul></section><section class="mt-10"><h2 class="text-2xl font-black tracking-tight">Diagnose the response in order</h2><ol class="mt-5 grid gap-3"><li class="panel p-5"><h3 class="font-bold">1. Start with a fresh regular request</h3><p class="text-base-content/70 mt-2 leading-7"> Fetch the public URL without browser cookies. Record the final URL, status, content type and response body. A 200 response is not enough if the body is a challenge page or an HTML error document. </p></li><li class="panel p-5"><h3 class="font-bold">2. Compare crawler-like requests</h3><p class="text-base-content/70 mt-2 leading-7"> Compare the same URL with common social crawler user agents. A changed status, title, body or redirect can reveal a user-agent rule, bot score, managed challenge or origin behavior that the browser path did not trigger. </p></li><li class="panel p-5"><h3 class="font-bold">3. Fetch every selected image</h3><p class="text-base-content/70 mt-2 leading-7"> Image hosts often have separate hotlink, WAF and cache rules. Follow redirects and verify the final status, TLS connection, MIME type and decoded dimensions for the exact URL in the metadata. </p></li><li class="panel p-5"><h3 class="font-bold">4. Find the matching security event</h3><p class="text-base-content/70 mt-2 leading-7"> Use your provider&#39;s event log to identify the product and rule that acted on the request. Do not disable several protections at once: that hides the cause and creates a wider exception than the preview needs. </p></li></ol></section><section class="mt-10"><h2 class="text-2xl font-black tracking-tight">Use verified identity where available</h2><div class="text-base-content/75 mt-4 space-y-4 leading-7"><p> A user-agent string is a claim, not proof. Anyone can send a request that says it is a social crawler. Prefer provider-maintained known-bot or verified-bot signals that use published IP ranges, reverse DNS or signed identity. </p><p> Cloudflare exposes the known-bot field <code>cf.client.bot</code> on all plans and the more specific <code>cf.bot_management.verified_bot</code> with Bot Management. Its WAF guidance shows how to exclude known or verified bots from a challenge rule. Apply the exception before the rule that would block or challenge the request. </p><p> Cloudflare also documents an important limitation: Bot Fight Mode cannot be skipped by a custom rule. Super Bot Fight Mode and Ruleset Engine phases can be skipped selectively. Check which feature generated the event before assuming a custom allow rule will take effect. </p></div><pre class="bg-base-300/60 mt-5 overflow-auto rounded-box p-4 text-sm leading-6"><code>(not cf.client.bot and ip.src.country in {&quot;US&quot; &quot;MX&quot;})</code></pre><p class="text-base-content/60 mt-3 text-sm leading-6"> This is Cloudflare&#39;s documented pattern for challenging selected traffic while excluding known bots. Adapt the countries, action and scope to your own policy; do not paste a rule without reading the matching security events. </p></section><section class="mt-10"><h2 class="text-2xl font-black tracking-tight">Keep the exception narrow</h2><div class="text-base-content/75 mt-4 space-y-4 leading-7"><p> A public marketing page and its preview image do not need the same controls as login, checkout or API routes. Scope an exception to verified traffic and only the phases or paths that block the public asset. Keep rate limits and application security in place where they are still useful. </p><p>

1 Do not allow every request with <code>facebook</code>, <code>twitter</code> or <code>linkedin</code> in its user agent. Do not allow a broad IP range copied from an old article. Bot ranges and signatures change, and a stale list can both block the real crawler and admit unrelated traffic. </p><p> After changing the rule, rerun the regular and crawler-like checks, inspect the security log again, and use the platform&#39;s official inspector when one exists. A PreviewChecks simulation can expose a risky policy, but it does not originate from the platform&#39;s verified network and cannot prove that platform&#39;s private fetch path. </p></div></section><section class="panel mt-10 p-5 sm:p-6"><h2 class="text-2xl font-black tracking-tight">Release checklist</h2><ul class="text-base-content/75 mt-4 list-disc space-y-2 pl-5 leading-7"><li>HTML and preview images work without cookies, JavaScript or authentication.</li><li>Regular and crawler-like requests receive the intended content type and body.</li><li>The exact blocking security event and rule are known.</li><li>The exception uses verified identity where the provider supports it.</li><li> The exception is limited by path, phase and action instead of disabling protection. </li><li>The result is retested from outside the browser session.</li></ul><a class="btn btn-primary mt-5" href="/">Compare crawler access</a></section><section class="mt-10 border-t border-base-300 pt-6"><h2 class="text-lg font-bold">Primary references</h2><ul class="text-base-content/70 mt-3 list-disc space-y-2 pl-5 leading-7"><li><a class="link link-hover" href="https://developers.cloudflare.com/waf/custom-rules/use-cases/allow-traffic-from-verified-bots/" target="_blank" rel="noopener noreferrer">Cloudflare: allow traffic from verified bots</a></li><li><a class="link link-hover" href="https://developers.cloudflare.com/bots/concepts/bot/verified-bots/" target="_blank" rel="noopener noreferrer">Cloudflare: how verified bots are identified</a></li><li><a class="link link-hover" href="https://developers.cloudflare.com/waf/feature-interoperability/" target="_blank" rel="noopener noreferrer">Cloudflare: security feature interoperability and skip limits</a></li></ul><p class="text-base-content/55 mt-3 text-sm">Reviewed July 14, 2026.</p></section></article><footer class="border-base-300 bg-base-100 border-t"><div class="mx-auto grid w-full max-w-5xl gap-8 px-4 py-8 text-sm sm:grid-cols-2 sm:px-6 lg:grid-cols-[minmax(180px,0.7fr)_minmax(150px,0.6fr)_minmax(200px,1fr)]"><div class="space-y-3"><a class="link link-hover font-semibold" href="/">PreviewChecks</a></div><nav class="flex flex-col items-start gap-2" aria-label="Site links"><p class="text-sm font-semibold text-base-content/70">Resources</p><!--[--><a class="link link-hover" href="/api">API</a><a class="link link-hover" href="/faq">FAQ</a><a class="link link-hover" href="https://comstud.io/privacy-policy">Privacy</a><a class="link link-hover" href="https://comstud.io/terms-and-conditions">Terms</a><a class="link link-hover" href="/guides/open-graph-image-order">Open Graph image order</a><a class="link link-hover" href="/guides/crawler-access">Crawler access</a><a class="link link-hover" href="/guides/pinterest-images">Pinterest images</a><!--]--></nav><nav class="flex flex-col items-start gap-2" aria-label="Other CoMStudio products"><p class="text-sm font-semibold text-base-content/70">Other products</p><!--[--><a class="link link-hover" href="https://andocs.io">AnDocs</a><a class="link link-hover" href="https://deplog.dev">DepLog</a><a class="link link-hover" href="https://dobrodesk.com">DobroDesk</a><a class="link link-hover" href="https://getnow.date">GetNowDate</a><a class="link link-hover" href="https://krynia.com">Krynia</a><a class="link link-hover" href="https://renderlog.com">RenderLog</a><a class="link link-hover" href="https://comstud.io/platform">CoMStudio Platform</a><!--]--></nav></div><div class="border-base-300/70 border-t"><div class="text-base-content/60 mx-auto w-full max-w-5xl px-4 py-5 text-xs sm:px-6"><p>© 2026 CoMStudio OÜ</p></div></div></footer></main></div><div id="teleports"></div>

1<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"be19e43e379b4421a16ad7b232896bea","r":1,"spa":2}' crossorigin="anonymous"></script>

1
2</body></html>