https://www.thechannelco.com/scripts/ffetch.js

1/*
2 * Copyright 2023 Adobe. All rights reserved.
3 * This file is licensed to you under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License. You may obtain a copy
5 * of the License at http://www.apache.org/licenses/LICENSE-2.0
6 *
7 * Unless required by applicable law or agreed to in writing, software distributed under
8 * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9 * OF ANY KIND, either express or implied. See the License for the specific language
10 * governing permissions and limitations under the License.
11 */
12
13/* eslint-disable no-restricted-syntax,  no-await-in-loop */
14
15async function* request(url, context) {
16  const {
17    chunkSize, cacheReload, sheetName, fetch,
18  } = context;
19  for (let offset = 0, total = Infinity; offset < total; offset += chunkSize) {
20    const params = new URLSearchParams(`offset=${offset}&limit=${chunkSize}`);
21    if (sheetName) params.append('sheet', sheetName);
22    const resp = await fetch(`${url}?${params.toString()}`, { cache: cacheReload ? 'reload' : 'default' });
23    if (resp.ok) {
24      const json = await resp.json();
25      total = json.total;
26      context.total = total;
27      for (const entry of json.data) yield entry;
28    } else {
29      return;
30    }
31  }
32}
33
34// Operations:
35
36function withFetch(upstream, context, fetch) {
37  context.fetch = fetch;
38  return upstream;
39}
40
41function withHtmlParser(upstream, context, parseHtml) {
42  context.parseHtml = parseHtml;
43  return upstream;
44}
45
46function chunks(upstream, context, chunkSize) {
47  context.chunkSize = chunkSize;
48  return upstream;
49}
50
51function sheet(upstream, context, sheetName) {
52  context.sheetName = sheetName;
53  return upstream;
54}
55
56async function* skip(upstream, context, from) {
57  let skipped = 0;
58  for await (const entry of upstream) {
59    if (skipped < from) {
60      skipped += 1;
61    } else {
62      yield entry;
63    }
64  }
65}
66
67async function* limit(upstream, context, aLimit) {
68  let yielded = 0;
69  for await (const entry of upstream) {
70    yield entry;
71    yielded += 1;
72    if (yielded === aLimit) {
73      return;
74    }
75  }
76}
77
78async function* map(upstream, context, fn, maxInFlight = 5) {
79  const promises = [];
80  for await (let entry of upstream) {
81    promises.push(fn(entry));
82    if (promises.length === maxInFlight) {
83      for (entry of promises) {
84        entry = await entry;
85        if (entry) yield entry;
86      }
87      promises.splice(0, promises.length);
88    }
89  }
90  for (let entry of promises) {
91    entry = await entry;
92    if (entry) yield entry;
93  }
94}
95
96async function* filter(upstream, context, fn) {
97  for await (const entry of upstream) {
98    if (fn(entry)) {
99      yield entry;
100    }
101  }
102}
103
104function slice(upstream, context, from, to) {
105  return limit(skip(upstream, context, from), context, to - from);
106}
107
108function follow(upstream, context, name, newName, maxInFlight = 5) {
109  const { fetch, parseHtml } = context;
110  return map(upstream, context, async (entry) => {
111    const value = entry[name];
112    if (value) {
113      const resp = await fetch(value);
114      return { ...entry, [newName || name]: resp.ok ? parseHtml(await resp.text()) : null };
115    }
116    return entry;
117  }, maxInFlight);
118}
119
120async function all(upstream) {
121  const result = [];
122  for await (const entry of upstream) {
123    result.push(entry);
124  }
125  return result;
126}
127
128async function first(upstream) {
129  /* eslint-disable-next-line no-unreachable-loop */
130  for await (const entry of upstream) {
131    return entry;
132  }
133  return null;
134}
135
136// Helper
137
138function assignOperations(generator, context) {
139  // operations that return a new generator
140  function createOperation(fn) {
141    return (...rest) => assignOperations(fn.apply(null, [generator, context, ...rest]), context);
142  }
143  const operations = {
144    skip: createOperation(skip),
145    limit: createOperation(limit),
146    slice: createOperation(slice),
147    map: createOperation(map),
148    filter: createOperation(filter),
149    follow: createOperation(follow),
150  };
151
152  // functions that either return the upstream generator or no generator at all
153  const functions = {
154    chunks: chunks.bind(null, generator, context),
155    all: all.bind(null, generator, context),
156    first: first.bind(null, generator, context),
157    withFetch: withFetch.bind(null, generator, context),
158    withHtmlParser: withHtmlParser.bind(null, generator, context),
159    sheet: sheet.bind(null, generator, context),
160  };
161
162  Object.assign(generator, operations, functions);
163  Object.defineProperty(generator, 'total', { get: () => context.total });
164  return generator;
165}
166
167export default function ffetch(url) {
168  let chunkSize = 255;
169  let cacheReload = false;

170  const fetch = (...rest) => window.fetch.apply(null, rest);
171  const parseHtml = (html) => new window.DOMParser().parseFromString(html, 'text/html');
172
173  try {
174    if ('connection' in window.navigator && window.navigator.connection.saveData === true) {
175      // request smaller chunks in save data mode
176      chunkSize = 64;
177    }
178    // detect page reloads and set cacheReload to true
179    const entries = performance.getEntriesByType('navigation');
180    const reloads = entries.filter((entry) => entry.type === 'reload');
181    if (reloads.length > 0) cacheReload = true;
182  } catch (e) { /* ignore */ }
183
184  const context = {
185    chunkSize, cacheReload, fetch, parseHtml,
186  };
187  const generator = request(url, context);
188
189  return assignOperations(generator, context);
190}