https://www.mk.ru/sw/sw-warmup-fetcher.js

1'use strict';
2
3async function warmupUrls(urls, concurrency) {
4  let success = 0;
5  let failed = 0;
6  let cursor = 0;
7  const reasonCounts = {};
8  const cachedUrls = [];
9  const failedUrls = [];
10
11  async function worker() {
12    while (cursor < urls.length) {
13      const currentIndex = cursor;
14      cursor += 1;
15      const currentUrl = urls[currentIndex];
16
17      try {
18        const response = await fetchWithTimeout(new Request(currentUrl, {
19          method: 'GET',
20          cache: 'no-cache',
21          credentials: 'same-origin',
22          headers: {
23            Accept: 'text/html',
24          },
25        }), WARMUP_REQUEST_TIMEOUT_MS);
26
27        if (!isCacheableHtmlResponse(response)) {
28          failed += 1;
29          const reason = buildWarmupReason(
30            WARMUP_REASON_CODES.FETCH_NOT_CACHEABLE,
31            response ? response.status : 'no_response'
32          );
33          log('debug', 'warmup.url.skip_not_cacheable', {
34            url: currentUrl,
35            status: response ? response.status : 'no_response',
36            reasonCode: reason.code,
37            reasonDetails: reason.details,
38          });
39          failedUrls.push({
40            url: currentUrl,
41            reasonCode: reason.code,
42            reasonDetails: reason.details,
43          });
44          reasonCounts[reason.code] = (reasonCounts[reason.code] || 0) + 1;
45          continue;
46        }
47
48        const cacheKey = normalizeHtmlCacheKey(currentUrl);
49        await cacheHtmlResponse(cacheKey, response.clone());
50        success += 1;
51        cachedUrls.push({
52          url: currentUrl,
53          cacheKey,
54          status: response.status,
55        });
56      } catch (error) {
57        failed += 1;
58        const errorMessage = String((error && error.message) || error);
59        const reasonCode = /^network_timeout_/i.test(errorMessage)
60          ? WARMUP_REASON_CODES.FETCH_TIMEOUT
61          : WARMUP_REASON_CODES.FETCH_NETWORK_ERROR;
62        const reason = buildWarmupReason(reasonCode, errorMessage);
63        log('debug', 'warmup.url.fail', {
64          url: currentUrl,
65          reasonCode: reason.code,
66          reasonDetails: reason.details,
67        });
68        failedUrls.push({
69          url: currentUrl,
70          reasonCode: reason.code,
71          reasonDetails: reason.details,
72        });
73        reasonCounts[reason.code] = (reasonCounts[reason.code] || 0) + 1;
74      }
75    }
76  }
77
78  const workers = [];
79  const workerCount = Math.max(1, Math.min(concurrency, urls.length || 1));
80  for (let i = 0; i < workerCount; i += 1) {
81    workers.push(worker());
82  }
83  await Promise.all(workers);
84
85  return { success, failed, reasonCounts, cachedUrls, failedUrls };
86}